Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod runtime_payloads;
4mod runtime_uploads;
5
6use std::{
7    any::{type_name, Any, TypeId},
8    collections::{BTreeMap, HashMap},
9    future::Future,
10    io::{self, Read},
11    pin::Pin,
12    sync::{
13        atomic::{AtomicBool, Ordering},
14        Arc, Mutex, OnceLock,
15    },
16    task::{Context as TaskContext, Poll},
17    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
18};
19
20use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
21use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
22use chrono::DateTime;
23use futures_util::{future::OptionFuture, task::noop_waker_ref};
24use serde::{
25    de::DeserializeOwned,
26    ser::{SerializeMap, SerializeSeq},
27    Deserialize, Deserializer, Serialize, Serializer,
28};
29pub use serde_json::{json, Value};
30use sha2::{Digest, Sha256};
31use thiserror::Error;
32pub use uuid::Uuid;
33
34pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
35/// First additive worker protocol that defines portable worker-affinity features.
36pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
37pub const CONTROL_PLANE_VERSION: &str = "2";
38pub const DEFAULT_CODEC: &str = "avro";
39pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
40/// Worker-registration capability for authored condition-wait occurrence identity.
41pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
42    "condition_wait_occurrence_identity";
43/// Worker-registration capability for portable memo upserts.
44pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
45/// Worker-registration capability for server-routed read-only queries.
46pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
47/// Worker-registration capability for canonical typed search attributes.
48pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
49/// Worker-registration capability for synchronous workflow updates.
50pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
51/// Worker-registration capability for durable named input streams.
52pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
53/// Worker-registration capability for persisted first-completion selection.
54pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
55pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
56pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
57pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
58pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
59pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
60/// First additive worker protocol that defines query-task transport.
61pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
62/// First additive worker protocol that defines typed search-attribute upserts.
63pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
64/// First additive worker protocol that defines portable memo upserts.
65pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
66/// First additive worker protocol that preserves declared search-attribute types.
67pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
68/// First additive worker protocol that defines external durable condition waits.
69pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
70/// First additive worker protocol that preserves authored condition-wait occurrences.
71pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
72/// First additive worker protocol that defines durable selection groups.
73pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
74
75pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
76    let Some((major, minor)) = version.split_once('.') else {
77        return false;
78    };
79    major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
80}
81
82fn validate_user_signal_name(signal_name: &str) -> Result<()> {
83    if signal_name == MESSAGE_STREAM_SIGNAL {
84        return Err(Error::Codec(format!(
85            "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
86        )));
87    }
88    Ok(())
89}
90
91const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
92const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
93    "Workflow task waiting for scheduled history.";
94const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
95const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
96const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
97const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
98const MAX_MEMO_ENTRIES: usize = 100;
99const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
100const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
101
102const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
103    "lease_expired",
104    "query_task_not_found",
105    "query_task_not_leased",
106    "query_task_timed_out",
107];
108
109/// Truthful service-worker manifest for features this SDK currently refuses.
110pub fn portable_worker_affinity_capability_manifest() -> Value {
111    json!({
112        "local_activities": {
113            "supported": false,
114            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
115            "reason": "rust_worker_does_not_execute_record_local_activity",
116        },
117        "worker_sessions": {
118            "supported": false,
119            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
120            "reason": "rust_worker_has_no_typed_session_lifecycle",
121        },
122        "sticky_execution": {
123            "supported": false,
124            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
125            "reason": "rust_worker_uses_complete_durable_history_replay",
126        },
127    })
128}
129
130/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
131pub const AVRO_VALUE_SCHEMA_JSON: &str =
132    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
133pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
134pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
135const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
136
137static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
138static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
139    OnceLock::new();
140
141#[derive(Clone, Copy)]
142enum RequestProtocol {
143    ControlPlane,
144    Worker(&'static str),
145}
146
147pub type Result<T> = std::result::Result<T, Error>;
148
149#[derive(Debug, Error)]
150pub enum Error {
151    #[error("transport error: {0}")]
152    Transport(#[from] reqwest::Error),
153    #[error(
154        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
155    )]
156    InvalidBaseUrl,
157    #[error("json error: {0}")]
158    Json(#[from] serde_json::Error),
159    #[error("http {status}: {body}")]
160    Http {
161        status: reqwest::StatusCode,
162        body: String,
163    },
164    #[error("codec error: {0}")]
165    Codec(String),
166    #[error(transparent)]
167    QueryFailed(QueryFailure),
168    #[error(transparent)]
169    Protocol(ProtocolFailure),
170    #[error(transparent)]
171    NonDeterministicReplay(ReplayFailure),
172    #[error(transparent)]
173    ChildWorkflowFailed(ChildWorkflowFailure),
174    #[error(transparent)]
175    ActivityFailed(ActivityFailure),
176    #[error(transparent)]
177    ParallelFailed(ParallelFailure),
178    #[error(transparent)]
179    SagaCompensationFailed(SagaCompensationFailure),
180    #[error(transparent)]
181    InvalidParallelGroup(ParallelGroupError),
182    #[error(transparent)]
183    DurableOperationCancelled(DurableOperationCancelled),
184    #[error(transparent)]
185    WorkflowCancellationRequested(WorkflowCancellationRequested),
186    #[error(transparent)]
187    WorkflowCommandRejected(WorkflowCommandRejection),
188    #[error(transparent)]
189    WorkflowFailed(WorkflowTerminalOutcome),
190    #[error(transparent)]
191    WorkflowCancelled(WorkflowTerminalOutcome),
192    #[error(transparent)]
193    WorkflowTerminated(WorkflowTerminalOutcome),
194    #[error(transparent)]
195    WorkflowTimedOut(WorkflowTerminalOutcome),
196    #[error(transparent)]
197    ActivityTaskRejected(ActivityTaskRejection),
198    #[error("workflow handler {0:?} is not registered")]
199    WorkflowNotRegistered(String),
200    #[error("activity handler {0:?} is not registered")]
201    ActivityNotRegistered(String),
202    #[error(
203        "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
204    )]
205    HandlerType {
206        handler_kind: HandlerKind,
207        handler_name: String,
208        value_kind: HandlerValueKind,
209        rust_type: &'static str,
210        message: String,
211    },
212    #[error("workflow future yielded without emitting a durable command")]
213    WorkflowYieldedWithoutCommand,
214    #[error(
215        "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
216    )]
217    MissingWorkflowCommandIdentity,
218    #[error("workflow state lock is poisoned")]
219    WorkflowStatePoisoned,
220    #[error("timer duration is too large for the worker protocol")]
221    TimerDurationOverflow,
222    #[error(transparent)]
223    InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
224    #[error(transparent)]
225    InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
226    #[error("operation timed out")]
227    Timeout,
228    #[error(
229        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
230    )]
231    MissingRoleCredentials {
232        role: &'static str,
233        opposite_role: &'static str,
234    },
235    #[error("worker loop error: {0}")]
236    WorkerLoop(String),
237    #[error(
238        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
239    )]
240    UnsupportedUpdateValidators { workflow_type: String },
241    #[error("{primary}; worker deregistration also failed: {deregistration}")]
242    WorkerShutdown {
243        primary: Box<Error>,
244        deregistration: Box<Error>,
245    },
246    #[error("invalid child workflow options: {0}")]
247    InvalidChildWorkflowOptions(String),
248    #[error("invalid workflow memo update: {0}")]
249    InvalidMemoUpdate(String),
250    #[error(
251        "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
252    )]
253    WorkflowMemoUpdatesUnavailable,
254    #[error(transparent)]
255    InvalidActivityOptions(ActivityOptionsError),
256    #[error(transparent)]
257    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
258    #[doc(hidden)]
259    #[error("workflow requested continue as new")]
260    ContinueAsNew(ContinueAsNewRequest),
261}
262
263/// Validation failure for a durable condition-wait definition.
264#[derive(Clone, Debug, Error, PartialEq, Eq)]
265pub enum ConditionWaitOptionsError {
266    #[error("condition_key must be non-empty")]
267    EmptyKey,
268    #[error("condition_definition_fingerprint must be non-empty")]
269    EmptyPredicateIdentity,
270    #[error("condition timeout is too large for the worker protocol")]
271    TimeoutOverflow,
272}
273
274/// Stable identity and optional durable timeout for a condition wait.
275///
276/// `predicate_identity` is recorded as the worker protocol's
277/// `condition_definition_fingerprint` and must change whenever predicate
278/// behavior changes. Prefer the [`wait_condition!`] macro when the predicate
279/// is written inline; it derives this identity from the predicate tokens.
280#[derive(Clone, Debug, PartialEq, Eq)]
281pub struct ConditionWaitOptions {
282    condition_key: String,
283    predicate_identity: String,
284    timeout: Option<Duration>,
285}
286
287impl ConditionWaitOptions {
288    pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
289        Self {
290            condition_key: condition_key.into(),
291            predicate_identity: predicate_identity.into(),
292            timeout: None,
293        }
294    }
295
296    pub fn timeout(mut self, timeout: Duration) -> Self {
297        self.timeout = Some(timeout);
298        self
299    }
300
301    fn validate(
302        &self,
303    ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
304        let condition_key = self.condition_key.trim();
305        if condition_key.is_empty() {
306            return Err(ConditionWaitOptionsError::EmptyKey);
307        }
308        let predicate_identity = self.predicate_identity.trim();
309        if predicate_identity.is_empty() {
310            return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
311        }
312        let timeout_seconds = self
313            .timeout
314            .map(|timeout| {
315                timeout
316                    .as_secs()
317                    .checked_add(u64::from(timeout.subsec_nanos() > 0))
318                    .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
319            })
320            .transpose()?;
321
322        Ok(ValidatedConditionWaitOptions {
323            condition_key: condition_key.to_string(),
324            predicate_identity: predicate_identity.to_string(),
325            timeout_seconds,
326        })
327    }
328}
329
330#[derive(Clone, Debug, PartialEq, Eq)]
331struct ValidatedConditionWaitOptions {
332    condition_key: String,
333    predicate_identity: String,
334    timeout_seconds: Option<u64>,
335}
336
337const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
338
339/// Unambiguous terminal result of a durable condition wait.
340#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
341#[serde(rename_all = "snake_case")]
342pub enum ConditionWaitResult {
343    Satisfied,
344    TimedOut,
345}
346
347impl ConditionWaitResult {
348    pub fn is_satisfied(self) -> bool {
349        self == Self::Satisfied
350    }
351
352    pub fn is_timed_out(self) -> bool {
353        self == Self::TimedOut
354    }
355}
356
357/// Build the stable condition definition identity used by [`wait_condition!`].
358#[doc(hidden)]
359pub fn __condition_definition_fingerprint(source: &str) -> String {
360    let mut digest = Sha256::new();
361    digest.update(b"durable-workflow-rust.wait-condition.v1\0");
362    digest.update(source.as_bytes());
363    format!("sha256:{:x}", digest.finalize())
364}
365
366/// Create a durable condition wait whose predicate definition is fingerprinted
367/// from its inline Rust tokens.
368///
369/// The returned [`ConditionWaitCall`] must be awaited. The timeout form is
370/// `wait_condition!(ctx, "approval", timeout: duration, || predicate)`.
371#[macro_export]
372macro_rules! wait_condition {
373    ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
374        $ctx.wait_condition(
375            $crate::ConditionWaitOptions::new(
376                $key,
377                $crate::__condition_definition_fingerprint(concat!(
378                    module_path!(),
379                    "\0",
380                    stringify!($predicate)
381                )),
382            )
383            .timeout($timeout),
384            $predicate,
385        )
386    }};
387    ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
388        $ctx.wait_condition(
389            $crate::ConditionWaitOptions::new(
390                $key,
391                $crate::__condition_definition_fingerprint(concat!(
392                    module_path!(),
393                    "\0",
394                    stringify!($predicate)
395                )),
396            ),
397            $predicate,
398        )
399    }};
400}
401
402const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
403const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
404const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
405const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
406const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
407
408/// Validation failure for a typed workflow search-attribute update.
409#[derive(Clone, Debug, Error, PartialEq, Eq)]
410pub enum SearchAttributeUpdateError {
411    #[error("search-attribute update requires at least one attribute")]
412    Empty,
413    #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
414    InvalidKey(String),
415    #[error("search-attribute update exceeds the limit of 100 attributes")]
416    TooManyAttributes,
417    #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
418    ValueTooLong {
419        key: String,
420        kind: &'static str,
421        limit: usize,
422    },
423    #[error(
424        "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
425    )]
426    EmptyString(String),
427    #[error("search attribute {0:?} has a non-finite float value")]
428    NonFiniteFloat(String),
429    #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
430    InvalidDateTime(String),
431    #[error("search-attribute update exceeds the 65536-byte protocol limit")]
432    PayloadTooLarge,
433}
434
435/// One public typed search-attribute value.
436#[derive(Clone, Debug, PartialEq)]
437pub enum SearchAttributeValue {
438    String(String),
439    Keyword(String),
440    KeywordList(Vec<String>),
441    Int(i64),
442    Float(f64),
443    Bool(bool),
444    DateTime(String),
445    Delete,
446}
447
448impl SearchAttributeValue {
449    fn type_name(&self) -> Option<&'static str> {
450        match self {
451            Self::String(_) => Some("string"),
452            Self::Keyword(_) => Some("keyword"),
453            Self::KeywordList(_) => Some("keyword_list"),
454            Self::Int(_) => Some("int"),
455            Self::Float(_) => Some("float"),
456            Self::Bool(_) => Some("bool"),
457            Self::DateTime(_) => Some("datetime"),
458            Self::Delete => None,
459        }
460    }
461
462    fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
463        let normalize_string = |value: String, kind: &'static str, limit: usize| {
464            let value = value.trim().to_string();
465            if value.is_empty() {
466                return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
467            }
468            if value.len() > limit {
469                return Err(SearchAttributeUpdateError::ValueTooLong {
470                    key: key.to_string(),
471                    kind,
472                    limit,
473                });
474            }
475            Ok(value)
476        };
477
478        match self {
479            Self::String(value) => Ok(Self::String(normalize_string(
480                value,
481                "string",
482                MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
483            )?)),
484            Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
485                value,
486                "keyword",
487                MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
488            )?)),
489            Self::KeywordList(values) => {
490                let values = values
491                    .into_iter()
492                    .map(|value| {
493                        let value = value.trim().to_string();
494                        if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
495                            return Err(SearchAttributeUpdateError::ValueTooLong {
496                                key: key.to_string(),
497                                kind: "keyword-list entry",
498                                limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
499                            });
500                        }
501                        Ok(value)
502                    })
503                    .collect::<std::result::Result<Vec<_>, _>>()?;
504                Ok(Self::KeywordList(values))
505            }
506            Self::Float(value) if !value.is_finite() => {
507                Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
508            }
509            Self::DateTime(value) => {
510                let value =
511                    normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
512                if DateTime::parse_from_rfc3339(&value).is_err() {
513                    return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
514                }
515                Ok(Self::DateTime(value))
516            }
517            value => Ok(value),
518        }
519    }
520
521    fn into_json(self) -> Value {
522        match self {
523            Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
524                Value::String(value)
525            }
526            Self::KeywordList(values) => {
527                Value::Array(values.into_iter().map(Value::String).collect())
528            }
529            Self::Int(value) => json!(value),
530            Self::Float(value) => json!(value),
531            Self::Bool(value) => json!(value),
532            Self::Delete => Value::Null,
533        }
534    }
535}
536
537/// Validated typed workflow-side search-attribute mutation.
538#[derive(Clone, Debug, Default, PartialEq)]
539pub struct SearchAttributeUpdate {
540    attributes: BTreeMap<String, SearchAttributeValue>,
541}
542
543impl SearchAttributeUpdate {
544    pub fn new() -> Self {
545        Self::default()
546    }
547
548    pub fn set(
549        mut self,
550        key: impl Into<String>,
551        value: SearchAttributeValue,
552    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
553        let key = key.into();
554        validate_search_attribute_key(&key)?;
555        if !self.attributes.contains_key(&key)
556            && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
557        {
558            return Err(SearchAttributeUpdateError::TooManyAttributes);
559        }
560        self.attributes.insert(key.clone(), value.normalized(&key)?);
561        self.validate_size()?;
562        Ok(self)
563    }
564
565    pub fn string(
566        self,
567        key: impl Into<String>,
568        value: impl Into<String>,
569    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
570        self.set(key, SearchAttributeValue::String(value.into()))
571    }
572
573    pub fn keyword(
574        self,
575        key: impl Into<String>,
576        value: impl Into<String>,
577    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
578        self.set(key, SearchAttributeValue::Keyword(value.into()))
579    }
580
581    pub fn keyword_list<I, V>(
582        self,
583        key: impl Into<String>,
584        values: I,
585    ) -> std::result::Result<Self, SearchAttributeUpdateError>
586    where
587        I: IntoIterator<Item = V>,
588        V: Into<String>,
589    {
590        self.set(
591            key,
592            SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
593        )
594    }
595
596    pub fn int(
597        self,
598        key: impl Into<String>,
599        value: i64,
600    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
601        self.set(key, SearchAttributeValue::Int(value))
602    }
603
604    pub fn float(
605        self,
606        key: impl Into<String>,
607        value: f64,
608    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
609        self.set(key, SearchAttributeValue::Float(value))
610    }
611
612    pub fn bool(
613        self,
614        key: impl Into<String>,
615        value: bool,
616    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
617        self.set(key, SearchAttributeValue::Bool(value))
618    }
619
620    pub fn datetime(
621        self,
622        key: impl Into<String>,
623        value: impl Into<String>,
624    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
625        self.set(key, SearchAttributeValue::DateTime(value.into()))
626    }
627
628    pub fn delete(
629        self,
630        key: impl Into<String>,
631    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
632        self.set(key, SearchAttributeValue::Delete)
633    }
634
635    fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
636        let (attributes, _) = self.clone().into_wire_parts();
637        if serde_json::to_vec(&attributes)
638            .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
639            .unwrap_or(true)
640        {
641            return Err(SearchAttributeUpdateError::PayloadTooLarge);
642        }
643        Ok(())
644    }
645
646    fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
647        let mut attributes = serde_json::Map::new();
648        let mut attribute_types = BTreeMap::new();
649        for (key, value) in self.attributes {
650            if let Some(type_name) = value.type_name() {
651                attribute_types.insert(key.clone(), type_name.to_string());
652            }
653            attributes.insert(key, value.into_json());
654        }
655        (Value::Object(attributes), attribute_types)
656    }
657
658    fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
659        if self.attributes.is_empty() {
660            return Err(SearchAttributeUpdateError::Empty);
661        }
662        self.validate_size()
663    }
664}
665
666fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
667    let valid = !key.is_empty()
668        && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
669        && key
670            .bytes()
671            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
672    if valid {
673        Ok(())
674    } else {
675        Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
676    }
677}
678
679/// The registered handler family reported by [`Error::HandlerType`].
680#[derive(Clone, Copy, Debug, PartialEq, Eq)]
681pub enum HandlerKind {
682    Workflow,
683    Activity,
684}
685
686impl std::fmt::Display for HandlerKind {
687    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
688        formatter.write_str(match self {
689            Self::Workflow => "workflow",
690            Self::Activity => "activity",
691        })
692    }
693}
694
695/// Whether a typed handler failed to adapt its input or result.
696#[derive(Clone, Copy, Debug, PartialEq, Eq)]
697pub enum HandlerValueKind {
698    Input,
699    Result,
700}
701
702impl std::fmt::Display for HandlerValueKind {
703    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
704        formatter.write_str(match self {
705            Self::Input => "input",
706            Self::Result => "result",
707        })
708    }
709}
710
711/// The lifecycle command sent to a workflow execution.
712#[derive(Clone, Copy, Debug, PartialEq, Eq)]
713pub enum WorkflowCommandKind {
714    Cancel,
715    Terminate,
716}
717
718impl WorkflowCommandKind {
719    fn as_str(self) -> &'static str {
720        match self {
721            Self::Cancel => "cancel",
722            Self::Terminate => "terminate",
723        }
724    }
725}
726
727/// Optional structured fields for a cancellation or termination request.
728#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
729pub struct WorkflowCommandOptions {
730    #[serde(skip_serializing_if = "Option::is_none")]
731    pub reason: Option<String>,
732    #[serde(skip_serializing_if = "Option::is_none")]
733    pub request_id: Option<String>,
734}
735
736/// Server-enforced timeout policy for a workflow start.
737///
738/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
739/// only bounds how long the caller waits. A server deadline produces a terminal
740/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
741/// `run_timeout`.
742#[derive(Clone, Debug, PartialEq, Eq)]
743pub struct WorkflowStartOptions {
744    pub execution_timeout_seconds: u64,
745    pub run_timeout_seconds: u64,
746}
747
748impl Default for WorkflowStartOptions {
749    fn default() -> Self {
750        Self {
751            execution_timeout_seconds: 3600,
752            run_timeout_seconds: 600,
753        }
754    }
755}
756
757impl WorkflowStartOptions {
758    pub fn new() -> Self {
759        Self::default()
760    }
761
762    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
763        self.execution_timeout_seconds = seconds;
764        self
765    }
766
767    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
768        self.run_timeout_seconds = seconds;
769        self
770    }
771
772    fn validate(&self) -> Result<()> {
773        if self.execution_timeout_seconds == 0 {
774            return Err(Error::Codec(
775                "execution_timeout_seconds must be at least 1".to_string(),
776            ));
777        }
778        if self.run_timeout_seconds == 0 {
779            return Err(Error::Codec(
780                "run_timeout_seconds must be at least 1".to_string(),
781            ));
782        }
783        if self.run_timeout_seconds > self.execution_timeout_seconds {
784            return Err(Error::Codec(
785                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
786            ));
787        }
788
789        Ok(())
790    }
791}
792
793/// Optional routing overrides for a continue-as-new transition.
794///
795/// Omitted values retain the current workflow type and task queue. Server-owned
796/// instance metadata is not accepted here and is carried by the server.
797#[derive(Clone, Debug, Default, PartialEq, Eq)]
798pub struct ContinueAsNewOptions {
799    pub workflow_type: Option<String>,
800    pub task_queue: Option<String>,
801}
802
803impl ContinueAsNewOptions {
804    pub fn new() -> Self {
805        Self::default()
806    }
807
808    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
809        self.workflow_type = Some(workflow_type.into());
810        self
811    }
812
813    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
814        self.task_queue = Some(task_queue.into());
815        self
816    }
817
818    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
819        for (field, value) in [
820            ("workflow_type", self.workflow_type.as_deref()),
821            ("task_queue", self.task_queue.as_deref()),
822        ] {
823            if value.is_some_and(|value| value.trim().is_empty()) {
824                return Err(ContinueAsNewOptionsError {
825                    field,
826                    message: format!("{field} must not be empty"),
827                });
828            }
829        }
830        Ok(())
831    }
832}
833
834/// A stable validation error raised before a continue-as-new command is emitted.
835#[derive(Clone, Debug, Error, PartialEq, Eq)]
836#[error("invalid continue-as-new option {field}: {message}")]
837pub struct ContinueAsNewOptionsError {
838    pub field: &'static str,
839    pub message: String,
840}
841
842/// Public history-budget information attached to the current workflow task.
843#[derive(Clone, Debug, Default, PartialEq, Eq)]
844pub struct WorkflowHistoryBudget {
845    pub event_count: u64,
846    pub size_bytes: Option<u64>,
847    pub continue_as_new_recommended: bool,
848    pub pressure: Option<String>,
849}
850
851#[doc(hidden)]
852#[derive(Clone, Debug)]
853pub struct ContinueAsNewRequest {
854    arguments: AvroValue,
855    options: ContinueAsNewOptions,
856}
857
858impl WorkflowCommandOptions {
859    pub fn new() -> Self {
860        Self::default()
861    }
862
863    pub fn reason(mut self, reason: impl Into<String>) -> Self {
864        self.reason = Some(reason.into());
865        self
866    }
867
868    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
869        self.request_id = Some(request_id.into());
870        self
871    }
872}
873
874/// The accepted, machine-readable result of a lifecycle command.
875#[derive(Clone, Debug, PartialEq)]
876pub struct WorkflowCommandResult {
877    pub command: WorkflowCommandKind,
878    pub workflow_id: String,
879    pub run_id: Option<String>,
880    pub outcome: Option<String>,
881    pub reason: Option<String>,
882    pub command_status: Option<String>,
883    pub raw: Value,
884}
885
886/// A stable rejection returned by instance- or selected-run lifecycle commands.
887#[derive(Clone, Debug, Error)]
888#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
889pub struct WorkflowCommandRejection {
890    pub command: WorkflowCommandKind,
891    pub status: u16,
892    pub reason: String,
893    pub message: String,
894    pub workflow_id: String,
895    pub run_id: Option<String>,
896    pub target_scope: Option<String>,
897    pub body: Value,
898}
899
900/// Stable terminal categories returned by [`WorkflowHandle::result`].
901#[derive(Clone, Copy, Debug, PartialEq, Eq)]
902pub enum WorkflowTerminalKind {
903    Failed,
904    Cancelled,
905    Terminated,
906    TimedOut,
907}
908
909/// A typed terminal workflow outcome with durable identity and failure metadata.
910///
911/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
912/// of parsing its display representation. Fields remain `None` when an older
913/// server did not publish that metadata.
914#[derive(Clone, Debug, Error)]
915#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
916pub struct WorkflowTerminalOutcome {
917    pub kind: WorkflowTerminalKind,
918    pub workflow_id: String,
919    pub run_id: Option<String>,
920    pub reason: String,
921    pub failure_category: Option<String>,
922    pub failure_id: Option<String>,
923    pub exception_type: Option<String>,
924    pub exception_class: Option<String>,
925    pub non_retryable: Option<bool>,
926    pub message: Option<String>,
927    pub exception: Option<Value>,
928    pub raw: Value,
929}
930
931/// A worker-side activity settlement or heartbeat rejected by durable state.
932#[derive(Clone, Debug, Error)]
933#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
934pub struct ActivityTaskRejection {
935    pub operation: String,
936    pub status: u16,
937    pub reason: String,
938    pub task_id: String,
939    pub activity_attempt_id: String,
940    pub cancel_requested: bool,
941    pub can_continue: Option<bool>,
942    pub run_closed_reason: Option<String>,
943    pub body: Value,
944}
945
946/// Stable validation categories for [`ActivityOptions`].
947#[derive(Clone, Copy, Debug, PartialEq, Eq)]
948pub enum ActivityOptionsErrorKind {
949    EmptyTaskQueue,
950    EmptyRetryPolicy,
951    InvalidMaxAttempts,
952    BackoffWithoutRetryBudget,
953    TooManyBackoffIntervals,
954    InvalidBackoffCoefficient,
955    BackoffGenerationTooLarge,
956    BackoffOverflow,
957    EmptyNonRetryableErrorType,
958    TimeoutNotPositive,
959    TimeoutOverflow,
960    TimeoutOrder,
961}
962
963/// A machine-readable activity-options validation failure.
964#[derive(Clone, Debug, Error, PartialEq, Eq)]
965#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
966pub struct ActivityOptionsError {
967    pub kind: ActivityOptionsErrorKind,
968    pub field: Option<&'static str>,
969    pub message: String,
970}
971
972impl ActivityOptionsError {
973    fn new(
974        kind: ActivityOptionsErrorKind,
975        field: Option<&'static str>,
976        message: impl Into<String>,
977    ) -> Self {
978        Self {
979            kind,
980            field,
981            message: message.into(),
982        }
983    }
984}
985
986/// Stable terminal categories returned when an awaited activity does not succeed.
987#[derive(Clone, Copy, Debug, PartialEq, Eq)]
988pub enum ActivityFailureKind {
989    Failed,
990    Cancelled,
991    TimedOut,
992}
993
994/// A stable, machine-readable terminal activity failure.
995///
996/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
997/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
998#[derive(Clone, Debug, Error)]
999#[error("activity failed ({reason}): {message}")]
1000pub struct ActivityFailure {
1001    pub kind: ActivityFailureKind,
1002    pub reason: String,
1003    pub message: String,
1004    pub activity_execution_id: Option<String>,
1005    pub activity_attempt_id: Option<String>,
1006    pub activity_type: Option<String>,
1007    pub activity_class: Option<String>,
1008    pub attempt_number: Option<u64>,
1009    pub failure_id: Option<String>,
1010    pub failure_category: Option<String>,
1011    pub timeout_kind: Option<String>,
1012    pub non_retryable: bool,
1013    pub exception_type: Option<String>,
1014    pub exception_class: Option<String>,
1015    pub code: Option<Value>,
1016    pub exception: Option<Value>,
1017}
1018
1019/// Stable terminal categories returned when an awaited child does not succeed.
1020#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1021pub enum ChildWorkflowFailureKind {
1022    Failed,
1023    Cancelled,
1024    Terminated,
1025}
1026
1027/// A stable, machine-readable child workflow failure delivered to its parent.
1028///
1029/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
1030/// of parsing the display message. Child and parent identifiers retain the
1031/// relationship recorded in durable history across worker restarts.
1032#[derive(Clone, Debug, Error)]
1033#[error("child workflow failed ({reason}): {message}")]
1034pub struct ChildWorkflowFailure {
1035    pub kind: ChildWorkflowFailureKind,
1036    pub reason: String,
1037    pub message: String,
1038    pub parent_workflow_id: Option<String>,
1039    pub parent_workflow_run_id: Option<String>,
1040    pub child_workflow_id: Option<String>,
1041    pub child_workflow_run_id: Option<String>,
1042    pub child_workflow_type: Option<String>,
1043    pub failure_id: Option<String>,
1044    pub failure_category: Option<String>,
1045    pub exception_type: Option<String>,
1046    pub exception_class: Option<String>,
1047    pub non_retryable: bool,
1048    pub code: Option<Value>,
1049    pub exception: Option<Value>,
1050}
1051
1052/// The identity of one durable workflow execution.
1053#[derive(Clone, Debug, PartialEq, Eq)]
1054pub struct WorkflowIdentity {
1055    pub workflow_id: Option<String>,
1056    pub run_id: Option<String>,
1057}
1058
1059/// A successful child result together with its durable parent-child identity.
1060#[derive(Clone, Debug, PartialEq)]
1061pub struct ChildWorkflowResult {
1062    pub parent: WorkflowIdentity,
1063    pub child: WorkflowIdentity,
1064    pub child_workflow_type: Option<String>,
1065    pub result: Value,
1066}
1067
1068/// Lossless successful child result for fixed Avro Value workflows.
1069#[derive(Clone, Debug, PartialEq)]
1070pub struct ChildWorkflowAvroResult {
1071    pub parent: WorkflowIdentity,
1072    pub child: WorkflowIdentity,
1073    pub child_workflow_type: Option<String>,
1074    pub result: AvroValue,
1075}
1076
1077/// Stable user-facing identity for one member of a durable selection group.
1078#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1079#[serde(untagged)]
1080pub enum SelectionKey {
1081    Index(usize),
1082    Name(String),
1083}
1084
1085impl From<usize> for SelectionKey {
1086    fn from(value: usize) -> Self {
1087        Self::Index(value)
1088    }
1089}
1090
1091impl From<String> for SelectionKey {
1092    fn from(value: String) -> Self {
1093        Self::Name(value)
1094    }
1095}
1096
1097impl From<&str> for SelectionKey {
1098    fn from(value: &str) -> Self {
1099        Self::Name(value.to_string())
1100    }
1101}
1102
1103/// Typed result of explicitly awaiting a cancelled non-winning operation.
1104#[derive(Clone, Debug, Error, PartialEq, Eq)]
1105#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1106pub struct DurableOperationCancelled {
1107    pub selection_group_id: String,
1108    pub member_key: SelectionKey,
1109    pub member_index: usize,
1110    pub operation_kind: String,
1111    pub operation_identity: String,
1112}
1113
1114/// Stable identity for one enclosing deterministic parallel group.
1115///
1116/// The same fields are attached to every ordinary activity, timer, or child
1117/// workflow command in the group. Nested leaves carry an outer-to-inner path;
1118/// no Rust-specific wire command is introduced.
1119#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1120pub struct ParallelGroupMetadata {
1121    pub parallel_group_id: String,
1122    pub parallel_group_kind: String,
1123    pub parallel_group_base_sequence: u64,
1124    pub parallel_group_size: usize,
1125    pub parallel_group_index: usize,
1126    #[serde(default, skip_serializing_if = "Option::is_none")]
1127    pub parallel_group_mode: Option<String>,
1128    #[serde(default, skip_serializing_if = "Option::is_none")]
1129    pub selection_member_key: Option<SelectionKey>,
1130    #[serde(default, skip_serializing_if = "Option::is_none")]
1131    pub selection_member_index: Option<usize>,
1132    #[serde(default, skip_serializing_if = "Option::is_none")]
1133    pub selection_member_base_sequence: Option<u64>,
1134    #[serde(default, skip_serializing_if = "Option::is_none")]
1135    pub selection_member_size: Option<usize>,
1136    #[serde(default, skip_serializing_if = "Option::is_none")]
1137    pub selection_member_kind: Option<String>,
1138}
1139
1140/// One input-ordered result returned by [`WorkflowContext::parallel`].
1141#[derive(Clone, Debug, PartialEq)]
1142pub enum ParallelResult {
1143    Activity(Value),
1144    ChildWorkflow(ChildWorkflowResult),
1145    Timer,
1146    Signal(Vec<Value>),
1147    Condition(ConditionWaitResult),
1148    Group(Vec<ParallelResult>),
1149}
1150
1151/// Lossless fixed-Avro counterpart to [`ParallelResult`].
1152#[derive(Clone, Debug, PartialEq)]
1153pub enum ParallelAvroResult {
1154    Activity(AvroValue),
1155    ChildWorkflow(ChildWorkflowAvroResult),
1156    Timer,
1157    Signal(Vec<AvroValue>),
1158    Condition(ConditionWaitResult),
1159    Group(Vec<ParallelAvroResult>),
1160}
1161
1162impl ParallelAvroResult {
1163    fn into_json_result(self) -> Result<ParallelResult> {
1164        match self {
1165            Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1166            Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1167                parent: result.parent,
1168                child: result.child,
1169                child_workflow_type: result.child_workflow_type,
1170                result: result.result.into_json()?,
1171            })),
1172            Self::Timer => Ok(ParallelResult::Timer),
1173            Self::Signal(values) => Ok(ParallelResult::Signal(
1174                values
1175                    .into_iter()
1176                    .map(AvroValue::into_json)
1177                    .collect::<Result<Vec<_>>>()?,
1178            )),
1179            Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1180            Self::Group(results) => Ok(ParallelResult::Group(
1181                results
1182                    .into_iter()
1183                    .map(Self::into_json_result)
1184                    .collect::<Result<Vec<_>>>()?,
1185            )),
1186        }
1187    }
1188}
1189
1190/// One successful leaf retained when another parallel member failed.
1191#[derive(Clone, Debug, PartialEq)]
1192pub struct ParallelCompletion {
1193    pub member_path: Vec<usize>,
1194    pub result: ParallelResult,
1195}
1196
1197/// A deterministic join failed after some siblings had already completed.
1198///
1199/// `cause` retains the typed activity, child-workflow, cancellation, or codec
1200/// error. `completed` is declaration ordered and contains only durable
1201/// successes observed in the same replay. Late sibling completions can add
1202/// entries on a later replay without changing `member_path` or the selected
1203/// positional failure.
1204#[derive(Debug, Error)]
1205#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1206pub struct ParallelFailure {
1207    pub group_id: String,
1208    pub member_path: Vec<usize>,
1209    pub group_path: Vec<ParallelGroupMetadata>,
1210    pub completed: Vec<ParallelCompletion>,
1211    #[source]
1212    pub cause: Box<Error>,
1213}
1214
1215/// Stable validation error returned before an invalid group emits commands.
1216#[derive(Clone, Debug, Error, PartialEq, Eq)]
1217#[error("invalid deterministic parallel group ({reason}): {message}")]
1218pub struct ParallelGroupError {
1219    pub reason: &'static str,
1220    pub member_path: Vec<usize>,
1221    pub message: String,
1222}
1223
1224/// Cooperative workflow cancellation observed at an author-controlled point.
1225#[derive(Clone, Debug, Error, PartialEq, Eq)]
1226#[error("workflow cancellation was requested")]
1227pub struct WorkflowCancellationRequested;
1228
1229/// A forward saga failure followed by a terminal compensation failure.
1230#[derive(Debug, Error)]
1231#[error(
1232    "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1233)]
1234pub struct SagaCompensationFailure {
1235    pub initiating_failure: Box<Error>,
1236    pub compensation_failure: Box<Error>,
1237    pub compensation_activity_type: String,
1238    pub compensation_registration_order: usize,
1239}
1240
1241/// Server behavior when a parent closes while its child is still open.
1242#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1243pub enum ParentClosePolicy {
1244    #[default]
1245    Abandon,
1246    RequestCancel,
1247    Terminate,
1248}
1249
1250impl ParentClosePolicy {
1251    fn as_str(self) -> &'static str {
1252        match self {
1253            Self::Abandon => "abandon",
1254            Self::RequestCancel => "request_cancel",
1255            Self::Terminate => "terminate",
1256        }
1257    }
1258}
1259
1260/// Durable retry policy for one child workflow invocation.
1261#[derive(Clone, Debug, Default, PartialEq, Eq)]
1262pub struct ChildWorkflowRetryPolicy {
1263    pub max_attempts: Option<u32>,
1264    pub backoff_seconds: Vec<u64>,
1265    pub non_retryable_error_types: Vec<String>,
1266}
1267
1268/// Options recorded with a child-workflow command.
1269///
1270/// The task queue is mandatory so routing is explicit and replay-stable.
1271#[derive(Clone, Debug, PartialEq, Eq)]
1272pub struct ChildWorkflowOptions {
1273    pub task_queue: String,
1274    pub parent_close_policy: ParentClosePolicy,
1275    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1276    pub execution_timeout_seconds: Option<u64>,
1277    pub run_timeout_seconds: Option<u64>,
1278}
1279
1280impl ChildWorkflowOptions {
1281    pub fn new(task_queue: impl Into<String>) -> Self {
1282        Self {
1283            task_queue: task_queue.into(),
1284            parent_close_policy: ParentClosePolicy::Abandon,
1285            retry_policy: None,
1286            execution_timeout_seconds: None,
1287            run_timeout_seconds: None,
1288        }
1289    }
1290
1291    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1292        self.parent_close_policy = policy;
1293        self
1294    }
1295
1296    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1297        self.retry_policy = Some(policy);
1298        self
1299    }
1300
1301    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1302        self.execution_timeout_seconds = Some(seconds);
1303        self
1304    }
1305
1306    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1307        self.run_timeout_seconds = Some(seconds);
1308        self
1309    }
1310}
1311
1312/// Backoff intervals for one durable activity retry policy.
1313#[derive(Clone, Debug, PartialEq, Eq)]
1314pub enum ActivityBackoff {
1315    /// Use these intervals between attempts. The server repeats the final
1316    /// interval if the retry budget contains more attempts than entries.
1317    Explicit(Vec<Duration>),
1318    /// Generate one interval for every retry using integer exponential growth.
1319    Exponential {
1320        initial_interval: Duration,
1321        coefficient: u32,
1322        maximum_interval: Option<Duration>,
1323    },
1324}
1325
1326/// Durable server-side retry policy for one activity execution.
1327#[derive(Clone, Debug, Default, PartialEq, Eq)]
1328pub struct ActivityRetryPolicy {
1329    pub max_attempts: Option<u32>,
1330    pub backoff: Option<ActivityBackoff>,
1331    pub non_retryable_error_types: Vec<String>,
1332}
1333
1334impl ActivityRetryPolicy {
1335    /// Start a policy with a finite attempt budget, including the first attempt.
1336    pub fn new(max_attempts: u32) -> Self {
1337        Self {
1338            max_attempts: Some(max_attempts),
1339            ..Self::default()
1340        }
1341    }
1342
1343    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1344        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1345        self
1346    }
1347
1348    pub fn exponential_backoff(
1349        mut self,
1350        initial_interval: Duration,
1351        coefficient: u32,
1352        maximum_interval: Option<Duration>,
1353    ) -> Self {
1354        self.backoff = Some(ActivityBackoff::Exponential {
1355            initial_interval,
1356            coefficient,
1357            maximum_interval,
1358        });
1359        self
1360    }
1361
1362    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1363        self.non_retryable_error_types.push(error_type.into());
1364        self
1365    }
1366
1367    pub fn non_retryable_error_types(
1368        mut self,
1369        error_types: impl IntoIterator<Item = impl Into<String>>,
1370    ) -> Self {
1371        self.non_retryable_error_types
1372            .extend(error_types.into_iter().map(Into::into));
1373        self
1374    }
1375}
1376
1377/// Options recorded atomically on one deterministic `schedule_activity` command.
1378///
1379/// Durations are rounded up to whole seconds when encoded, so the server never
1380/// receives a shorter timeout or backoff than the caller requested.
1381#[derive(Clone, Debug, Default, PartialEq, Eq)]
1382pub struct ActivityOptions {
1383    pub task_queue: Option<String>,
1384    pub retry_policy: Option<ActivityRetryPolicy>,
1385    pub start_to_close_timeout: Option<Duration>,
1386    pub schedule_to_start_timeout: Option<Duration>,
1387    pub schedule_to_close_timeout: Option<Duration>,
1388    pub heartbeat_timeout: Option<Duration>,
1389}
1390
1391impl ActivityOptions {
1392    pub fn new() -> Self {
1393        Self::default()
1394    }
1395
1396    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1397        self.task_queue = Some(task_queue.into());
1398        self
1399    }
1400
1401    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1402        self.retry_policy = Some(policy);
1403        self
1404    }
1405
1406    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1407        self.start_to_close_timeout = Some(timeout);
1408        self
1409    }
1410
1411    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1412        self.schedule_to_start_timeout = Some(timeout);
1413        self
1414    }
1415
1416    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1417        self.schedule_to_close_timeout = Some(timeout);
1418        self
1419    }
1420
1421    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1422        self.heartbeat_timeout = Some(timeout);
1423        self
1424    }
1425
1426    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1427        if self
1428            .task_queue
1429            .as_deref()
1430            .is_some_and(|queue| queue.trim().is_empty())
1431        {
1432            return Err(ActivityOptionsError::new(
1433                ActivityOptionsErrorKind::EmptyTaskQueue,
1434                Some("task_queue"),
1435                "task_queue must not be empty",
1436            ));
1437        }
1438
1439        for (field, value) in [
1440            ("start_to_close_timeout", self.start_to_close_timeout),
1441            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1442            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1443            ("heartbeat_timeout", self.heartbeat_timeout),
1444        ] {
1445            if value.is_some_and(|value| value.is_zero()) {
1446                return Err(ActivityOptionsError::new(
1447                    ActivityOptionsErrorKind::TimeoutNotPositive,
1448                    Some(field),
1449                    format!("{field} must be positive"),
1450                ));
1451            }
1452        }
1453
1454        validate_timeout_order(
1455            "heartbeat_timeout",
1456            self.heartbeat_timeout,
1457            "start_to_close_timeout",
1458            self.start_to_close_timeout,
1459        )?;
1460        validate_timeout_order(
1461            "start_to_close_timeout",
1462            self.start_to_close_timeout,
1463            "schedule_to_close_timeout",
1464            self.schedule_to_close_timeout,
1465        )?;
1466        validate_timeout_order(
1467            "schedule_to_start_timeout",
1468            self.schedule_to_start_timeout,
1469            "schedule_to_close_timeout",
1470            self.schedule_to_close_timeout,
1471        )?;
1472
1473        Ok(ValidatedActivityOptions {
1474            task_queue: self.task_queue.clone(),
1475            retry_policy: self
1476                .retry_policy
1477                .as_ref()
1478                .map(validate_activity_retry_policy)
1479                .transpose()?,
1480            start_to_close_timeout: timeout_seconds(
1481                "start_to_close_timeout",
1482                self.start_to_close_timeout,
1483            )?,
1484            schedule_to_start_timeout: timeout_seconds(
1485                "schedule_to_start_timeout",
1486                self.schedule_to_start_timeout,
1487            )?,
1488            schedule_to_close_timeout: timeout_seconds(
1489                "schedule_to_close_timeout",
1490                self.schedule_to_close_timeout,
1491            )?,
1492            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1493        })
1494    }
1495}
1496
1497/// A deferred durable leaf or nested group for [`WorkflowContext::parallel`].
1498///
1499/// Constructors capture arguments but perform no I/O. The join validates the
1500/// complete tree, attaches the existing parallel-group metadata to every
1501/// ordinary command, schedules all leaves, and then suspends.
1502pub enum ParallelOperation {
1503    Activity {
1504        activity_type: String,
1505        options: ActivityOptions,
1506        arguments: Result<AvroValue>,
1507    },
1508    ChildWorkflow {
1509        workflow_type: String,
1510        options: ChildWorkflowOptions,
1511        arguments: Result<AvroValue>,
1512    },
1513    Timer(Duration),
1514    Signal(String),
1515    Condition {
1516        options: ConditionWaitOptions,
1517        predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1518    },
1519    Group(Vec<ParallelOperation>),
1520}
1521
1522impl ParallelOperation {
1523    pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1524        Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1525    }
1526
1527    pub fn activity_with_options<T: Serialize>(
1528        activity_type: impl Into<String>,
1529        options: ActivityOptions,
1530        args: T,
1531    ) -> Self {
1532        Self::Activity {
1533            activity_type: activity_type.into(),
1534            options,
1535            arguments: AvroValue::from_serialize(&args),
1536        }
1537    }
1538
1539    pub fn child_workflow<T: Serialize>(
1540        workflow_type: impl Into<String>,
1541        options: ChildWorkflowOptions,
1542        args: T,
1543    ) -> Self {
1544        Self::ChildWorkflow {
1545            workflow_type: workflow_type.into(),
1546            options,
1547            arguments: AvroValue::from_serialize(&args),
1548        }
1549    }
1550
1551    pub fn timer(duration: Duration) -> Self {
1552        Self::Timer(duration)
1553    }
1554
1555    pub fn signal(signal_name: impl Into<String>) -> Self {
1556        Self::Signal(signal_name.into())
1557    }
1558
1559    pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1560    where
1561        F: Fn() -> Result<bool> + Send + 'static,
1562    {
1563        Self::Condition {
1564            options,
1565            predicate: Box::new(predicate),
1566        }
1567    }
1568
1569    pub fn group(operations: Vec<ParallelOperation>) -> Self {
1570        Self::Group(operations)
1571    }
1572}
1573
1574#[derive(Clone, Debug)]
1575struct ValidatedActivityOptions {
1576    task_queue: Option<String>,
1577    retry_policy: Option<Value>,
1578    start_to_close_timeout: Option<u64>,
1579    schedule_to_start_timeout: Option<u64>,
1580    schedule_to_close_timeout: Option<u64>,
1581    heartbeat_timeout: Option<u64>,
1582}
1583
1584fn validate_timeout_order(
1585    smaller_name: &'static str,
1586    smaller: Option<Duration>,
1587    larger_name: &'static str,
1588    larger: Option<Duration>,
1589) -> std::result::Result<(), ActivityOptionsError> {
1590    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1591        return Err(ActivityOptionsError::new(
1592            ActivityOptionsErrorKind::TimeoutOrder,
1593            Some(smaller_name),
1594            format!("{smaller_name} must be <= {larger_name}"),
1595        ));
1596    }
1597    Ok(())
1598}
1599
1600fn timeout_seconds(
1601    field: &'static str,
1602    value: Option<Duration>,
1603) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1604    value
1605        .map(|value| {
1606            activity_protocol_seconds(value).ok_or_else(|| {
1607                ActivityOptionsError::new(
1608                    ActivityOptionsErrorKind::TimeoutOverflow,
1609                    Some(field),
1610                    format!("{field} is too large for the worker protocol"),
1611                )
1612            })
1613        })
1614        .transpose()
1615}
1616
1617fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1618    value
1619        .as_secs()
1620        .checked_add(u64::from(value.subsec_nanos() > 0))
1621}
1622
1623fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1624    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1625}
1626
1627fn validate_activity_retry_policy(
1628    policy: &ActivityRetryPolicy,
1629) -> std::result::Result<Value, ActivityOptionsError> {
1630    if policy.max_attempts.is_none()
1631        && policy.backoff.is_none()
1632        && policy.non_retryable_error_types.is_empty()
1633    {
1634        return Err(ActivityOptionsError::new(
1635            ActivityOptionsErrorKind::EmptyRetryPolicy,
1636            Some("retry_policy"),
1637            "retry_policy must configure at least one field",
1638        ));
1639    }
1640    if policy.max_attempts == Some(0) {
1641        return Err(ActivityOptionsError::new(
1642            ActivityOptionsErrorKind::InvalidMaxAttempts,
1643            Some("retry_policy.max_attempts"),
1644            "max_attempts must be >= 1",
1645        ));
1646    }
1647    if policy
1648        .non_retryable_error_types
1649        .iter()
1650        .any(|error_type| error_type.trim().is_empty())
1651    {
1652        return Err(ActivityOptionsError::new(
1653            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1654            Some("retry_policy.non_retryable_error_types"),
1655            "non_retryable_error_types must not contain empty values",
1656        ));
1657    }
1658
1659    let backoff_seconds = match &policy.backoff {
1660        None => None,
1661        Some(backoff) => {
1662            let max_attempts = policy.max_attempts.ok_or_else(|| {
1663                ActivityOptionsError::new(
1664                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1665                    Some("retry_policy.backoff"),
1666                    "backoff requires max_attempts",
1667                )
1668            })?;
1669            let retry_count = max_attempts.saturating_sub(1) as usize;
1670            let intervals = match backoff {
1671                ActivityBackoff::Explicit(intervals) => {
1672                    if intervals.len() > retry_count {
1673                        return Err(ActivityOptionsError::new(
1674                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
1675                            Some("retry_policy.backoff"),
1676                            "backoff interval count must not exceed max_attempts - 1",
1677                        ));
1678                    }
1679                    intervals.clone()
1680                }
1681                ActivityBackoff::Exponential {
1682                    initial_interval,
1683                    coefficient,
1684                    maximum_interval,
1685                } => {
1686                    if *coefficient < 1 {
1687                        return Err(ActivityOptionsError::new(
1688                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1689                            Some("retry_policy.backoff.coefficient"),
1690                            "backoff coefficient must be >= 1",
1691                        ));
1692                    }
1693                    if retry_count > 10_000 {
1694                        return Err(ActivityOptionsError::new(
1695                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1696                            Some("retry_policy.max_attempts"),
1697                            "generated backoff supports at most 10000 retry intervals",
1698                        ));
1699                    }
1700                    let mut current = *initial_interval;
1701                    let mut intervals = Vec::with_capacity(retry_count);
1702                    for _ in 0..retry_count {
1703                        let interval = maximum_interval
1704                            .map(|maximum| current.min(maximum))
1705                            .unwrap_or(current);
1706                        intervals.push(interval);
1707                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
1708                            break;
1709                        }
1710                        current = current.checked_mul(*coefficient).ok_or_else(|| {
1711                            ActivityOptionsError::new(
1712                                ActivityOptionsErrorKind::BackoffOverflow,
1713                                Some("retry_policy.backoff"),
1714                                "generated backoff interval overflowed",
1715                            )
1716                        })?;
1717                    }
1718                    intervals
1719                }
1720            };
1721            Some(
1722                intervals
1723                    .into_iter()
1724                    .map(|interval| {
1725                        activity_protocol_seconds(interval).ok_or_else(|| {
1726                            ActivityOptionsError::new(
1727                                ActivityOptionsErrorKind::BackoffOverflow,
1728                                Some("retry_policy.backoff"),
1729                                "backoff interval is too large for the worker protocol",
1730                            )
1731                        })
1732                    })
1733                    .collect::<std::result::Result<Vec<_>, _>>()?,
1734            )
1735        }
1736    };
1737
1738    let mut encoded = serde_json::Map::new();
1739    if let Some(max_attempts) = policy.max_attempts {
1740        encoded.insert("max_attempts".to_string(), json!(max_attempts));
1741    }
1742    if let Some(backoff_seconds) = backoff_seconds {
1743        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1744    }
1745    if !policy.non_retryable_error_types.is_empty() {
1746        let mut canonical_error_types = Vec::new();
1747        for error_type in policy
1748            .non_retryable_error_types
1749            .iter()
1750            .map(|error_type| error_type.trim())
1751        {
1752            if !canonical_error_types.contains(&error_type) {
1753                canonical_error_types.push(error_type);
1754            }
1755        }
1756        encoded.insert(
1757            "non_retryable_error_types".to_string(),
1758            json!(canonical_error_types),
1759        );
1760    }
1761    Ok(Value::Object(encoded))
1762}
1763
1764/// A stable, machine-readable failure raised when workflow code no longer
1765/// reconstructs the durable command stream recorded in history.
1766#[derive(Clone, Debug, Error)]
1767#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1768pub struct ReplayFailure {
1769    pub reason: String,
1770    pub sequence: Option<u64>,
1771    pub expected: Option<String>,
1772    pub actual: Option<String>,
1773    pub message: String,
1774}
1775
1776impl ReplayFailure {
1777    fn new(
1778        reason: impl Into<String>,
1779        sequence: Option<u64>,
1780        expected: Option<String>,
1781        actual: Option<String>,
1782        message: impl Into<String>,
1783    ) -> Self {
1784        Self {
1785            reason: reason.into(),
1786            sequence,
1787            expected,
1788            actual,
1789            message: message.into(),
1790        }
1791    }
1792}
1793
1794/// A stable, machine-readable workflow query or query-task settlement failure.
1795#[derive(Clone, Debug, Error)]
1796#[error("query failed ({reason}, HTTP {status}): {message}")]
1797pub struct QueryFailure {
1798    pub status: u16,
1799    pub reason: String,
1800    pub message: String,
1801    pub body: Value,
1802}
1803
1804/// A stable failure returned when a server rejects an SDK protocol version.
1805#[derive(Clone, Debug, Error)]
1806#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1807pub struct ProtocolFailure {
1808    pub status: u16,
1809    pub reason: String,
1810    pub message: String,
1811    pub supported_version: Option<String>,
1812    pub requested_version: Option<String>,
1813    pub body: Value,
1814}
1815
1816#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1817pub struct PayloadEnvelope {
1818    pub codec: String,
1819    pub blob: String,
1820}
1821
1822impl PayloadEnvelope {
1823    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1824        encode_payload(value, DEFAULT_CODEC)
1825    }
1826
1827    /// Encode an explicit typed value, including the bytes branch that JSON
1828    /// serialization cannot represent.
1829    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1830        encode_avro_value(value)
1831    }
1832}
1833
1834/// Native adapter for the fixed language-neutral Avro Value schema.
1835#[derive(Clone, Debug)]
1836pub enum AvroValue {
1837    Null,
1838    Boolean(bool),
1839    Long(i64),
1840    Double(f64),
1841    Bytes(Vec<u8>),
1842    String(String),
1843    Array(Vec<AvroValue>),
1844    Map(BTreeMap<String, AvroValue>),
1845}
1846
1847impl PartialEq for AvroValue {
1848    fn eq(&self, other: &Self) -> bool {
1849        match (self, other) {
1850            (Self::Null, Self::Null) => true,
1851            (Self::Boolean(left), Self::Boolean(right)) => left == right,
1852            (Self::Long(left), Self::Long(right)) => left == right,
1853            (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1854            (Self::Bytes(left), Self::Bytes(right)) => left == right,
1855            (Self::String(left), Self::String(right)) => left == right,
1856            (Self::Array(left), Self::Array(right)) => left == right,
1857            (Self::Map(left), Self::Map(right)) => left == right,
1858            _ => false,
1859        }
1860    }
1861}
1862
1863impl AvroValue {
1864    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1865        Self::from_serde_value(
1866            serde_value::to_value(value).map_err(|error| {
1867                Error::Codec(format!("could not adapt value for Avro: {error}"))
1868            })?,
1869        )
1870    }
1871
1872    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1873        use serde_value::Value as SerdeValue;
1874
1875        match value {
1876            SerdeValue::Unit => Ok(Self::Null),
1877            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1878            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1879            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1880            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1881            SerdeValue::I64(value) => Ok(Self::Long(value)),
1882            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1883            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1884            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1885            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1886                Error::Codec(
1887                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1888                        .to_string(),
1889                )
1890            }),
1891            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1892            SerdeValue::F64(value) => Self::finite_double(value),
1893            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1894            SerdeValue::String(value) => Ok(Self::String(value)),
1895            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1896            SerdeValue::Option(None) => Ok(Self::Null),
1897            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1898                Self::from_serde_value(*value)
1899            }
1900            SerdeValue::Seq(values) => values
1901                .into_iter()
1902                .map(Self::from_serde_value)
1903                .collect::<Result<Vec<_>>>()
1904                .map(Self::Array),
1905            SerdeValue::Map(values) => values
1906                .into_iter()
1907                .map(|(key, value)| {
1908                    let SerdeValue::String(key) = key else {
1909                        return Err(Error::Codec(
1910                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1911                        ));
1912                    };
1913
1914                    Ok((key, Self::from_serde_value(value)?))
1915                })
1916                .collect::<Result<BTreeMap<_, _>>>()
1917                .map(Self::Map),
1918        }
1919    }
1920
1921    fn finite_double(value: f64) -> Result<Self> {
1922        if !value.is_finite() {
1923            return Err(Error::Codec(
1924                "non_finite_float: Avro Value doubles must be finite".to_string(),
1925            ));
1926        }
1927
1928        Ok(Self::Double(value))
1929    }
1930
1931    fn into_json(self) -> Result<Value> {
1932        match self {
1933            Self::Null => Ok(Value::Null),
1934            Self::Boolean(value) => Ok(Value::Bool(value)),
1935            Self::Long(value) => Ok(Value::Number(value.into())),
1936            Self::Double(value) => serde_json::Number::from_f64(value)
1937                .map(Value::Number)
1938                .ok_or_else(|| {
1939                    Error::Codec(
1940                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1941                    )
1942                }),
1943            Self::Bytes(value) => Ok(json!({
1944                "$type": "bytes",
1945                "base64": BASE64.encode(value),
1946            })),
1947            Self::String(value) => Ok(Value::String(value)),
1948            Self::Array(values) => values
1949                .into_iter()
1950                .map(Self::into_json)
1951                .collect::<Result<Vec<_>>>()
1952                .map(Value::Array),
1953            Self::Map(values) => values
1954                .into_iter()
1955                .map(|(key, value)| Ok((key, value.into_json()?)))
1956                .collect::<Result<serde_json::Map<_, _>>>()
1957                .map(Value::Object),
1958        }
1959    }
1960
1961    fn into_serde_value(self) -> serde_value::Value {
1962        use serde_value::Value as SerdeValue;
1963
1964        match self {
1965            Self::Null => SerdeValue::Unit,
1966            Self::Boolean(value) => SerdeValue::Bool(value),
1967            Self::Long(value) => SerdeValue::I64(value),
1968            Self::Double(value) => SerdeValue::F64(value),
1969            Self::Bytes(value) => SerdeValue::Bytes(value),
1970            Self::String(value) => SerdeValue::String(value),
1971            Self::Array(values) => {
1972                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1973            }
1974            Self::Map(values) => SerdeValue::Map(
1975                values
1976                    .into_iter()
1977                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1978                    .collect(),
1979            ),
1980        }
1981    }
1982
1983    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1984        self.into_serde_value().deserialize_into().map_err(|error| {
1985            Error::Codec(format!(
1986                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1987            ))
1988        })
1989    }
1990}
1991
1992impl Serialize for AvroValue {
1993    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1994    where
1995        S: Serializer,
1996    {
1997        match self {
1998            Self::Null => serializer.serialize_unit(),
1999            Self::Boolean(value) => serializer.serialize_bool(*value),
2000            Self::Long(value) => serializer.serialize_i64(*value),
2001            Self::Double(value) => serializer.serialize_f64(*value),
2002            Self::Bytes(value) => serializer.serialize_bytes(value),
2003            Self::String(value) => serializer.serialize_str(value),
2004            Self::Array(values) => {
2005                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2006                for value in values {
2007                    sequence.serialize_element(value)?;
2008                }
2009                sequence.end()
2010            }
2011            Self::Map(values) => {
2012                let mut map = serializer.serialize_map(Some(values.len()))?;
2013                for (key, value) in values {
2014                    map.serialize_entry(key, value)?;
2015                }
2016                map.end()
2017            }
2018        }
2019    }
2020}
2021
2022pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2023    let datum = avro_value_to_datum(value)?;
2024    let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2025        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2026    let mut bytes = Vec::with_capacity(datum.len() + 10);
2027    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2028    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2029    bytes.extend_from_slice(&datum);
2030    Ok(PayloadEnvelope {
2031        codec: DEFAULT_CODEC.to_string(),
2032        blob: BASE64.encode(bytes),
2033    })
2034}
2035
2036pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2037    if envelope.codec != DEFAULT_CODEC {
2038        return Err(unsupported_payload_codec(&envelope.codec));
2039    }
2040    decode_avro_value_blob(&envelope.blob)
2041}
2042
2043pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2044    let blob = match codec {
2045        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2046        other => return Err(unsupported_payload_codec(other)),
2047    };
2048
2049    Ok(PayloadEnvelope {
2050        codec: codec.to_string(),
2051        blob,
2052    })
2053}
2054
2055pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2056    match envelope.codec.as_str() {
2057        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2058        other => Err(unsupported_payload_codec(other)),
2059    }
2060}
2061
2062fn handler_type_error<T>(
2063    handler_kind: HandlerKind,
2064    handler_name: &str,
2065    value_kind: HandlerValueKind,
2066    message: impl Into<String>,
2067) -> Error {
2068    Error::HandlerType {
2069        handler_kind,
2070        handler_name: handler_name.to_string(),
2071        value_kind,
2072        rust_type: type_name::<T>(),
2073        message: message.into(),
2074    }
2075}
2076
2077fn decode_handler_input<T: DeserializeOwned>(
2078    arguments: AvroValue,
2079    handler_kind: HandlerKind,
2080    handler_name: &str,
2081) -> Result<T> {
2082    let argument = match arguments {
2083        AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2084            arguments.pop().expect("one typed handler argument")
2085        }
2086        AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2087        AvroValue::Array(arguments) => {
2088            return Err(handler_type_error::<T>(
2089                handler_kind,
2090                handler_name,
2091                HandlerValueKind::Input,
2092                format!(
2093                    "typed handlers accept one request value, but the task carried {} arguments",
2094                    arguments.len()
2095                ),
2096            ));
2097        }
2098        argument => argument,
2099    };
2100
2101    argument.deserialize().map_err(|error| {
2102        handler_type_error::<T>(
2103            handler_kind,
2104            handler_name,
2105            HandlerValueKind::Input,
2106            error.to_string(),
2107        )
2108    })
2109}
2110
2111fn encode_handler_result<T: Serialize>(
2112    result: &T,
2113    handler_kind: HandlerKind,
2114    handler_name: &str,
2115) -> Result<AvroValue> {
2116    AvroValue::from_serialize(result).map_err(|error| {
2117        handler_type_error::<T>(
2118            handler_kind,
2119            handler_name,
2120            HandlerValueKind::Result,
2121            error.to_string(),
2122        )
2123    })
2124}
2125
2126fn decode_handler_result<T: DeserializeOwned>(
2127    result: AvroValue,
2128    handler_kind: HandlerKind,
2129    handler_name: &str,
2130) -> Result<T> {
2131    result.deserialize().map_err(|error| {
2132        handler_type_error::<T>(
2133            handler_kind,
2134            handler_name,
2135            HandlerValueKind::Result,
2136            error.to_string(),
2137        )
2138    })
2139}
2140
2141#[cfg(test)]
2142fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2143    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2144}
2145
2146fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2147    validate_payload_codec(fallback_codec)?;
2148
2149    if value.is_null() {
2150        return Ok(Value::Null);
2151    }
2152
2153    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2154        return decode_blob(blob, codec);
2155    }
2156
2157    if let Some(blob) = value.as_str() {
2158        return decode_blob(blob, fallback_codec);
2159    }
2160
2161    Err(untagged_payload_value())
2162}
2163
2164fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2165    let envelope = match codec {
2166        DEFAULT_CODEC => encode_avro_value(value)?,
2167        other => return Err(unsupported_payload_codec(other)),
2168    };
2169    Ok(serde_json::to_value(envelope)?)
2170}
2171
2172fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2173    validate_payload_codec(fallback_codec)?;
2174
2175    if value.is_null() {
2176        return Ok(AvroValue::Null);
2177    }
2178
2179    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2180        validate_payload_codec(codec)?;
2181        return decode_avro_value_blob(blob);
2182    }
2183
2184    if let Some(blob) = value.as_str() {
2185        return match fallback_codec {
2186            DEFAULT_CODEC => decode_avro_value_blob(blob),
2187            other => Err(unsupported_payload_codec(other)),
2188        };
2189    }
2190
2191    Err(untagged_payload_value())
2192}
2193
2194fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2195    match value {
2196        AvroValue::Null => AvroValue::Array(Vec::new()),
2197        AvroValue::Array(_) => value,
2198        other => AvroValue::Array(vec![other]),
2199    }
2200}
2201
2202fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2203    match codec {
2204        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2205        other => Err(unsupported_payload_codec(other)),
2206    }
2207}
2208
2209fn validate_payload_codec(codec: &str) -> Result<()> {
2210    match codec {
2211        DEFAULT_CODEC => Ok(()),
2212        MISSING_TASK_PAYLOAD_CODEC => {
2213            Err(invalid_task_payload_codec("task payload_codec is missing"))
2214        }
2215        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2216        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2217            "task payload_codec must be a string",
2218        )),
2219        other => Err(unsupported_payload_codec(other)),
2220    }
2221}
2222
2223fn invalid_task_payload_codec(reason: &str) -> Error {
2224    Error::Codec(format!(
2225        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2226    ))
2227}
2228
2229fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2230    let Some(object) = value.as_object() else {
2231        return Ok(None);
2232    };
2233    if !object.contains_key("codec") && !object.contains_key("blob") {
2234        return Ok(None);
2235    }
2236
2237    let codec = object
2238        .get("codec")
2239        .and_then(Value::as_str)
2240        .ok_or_else(invalid_payload_envelope)?;
2241    validate_payload_codec(codec)?;
2242    let blob = object
2243        .get("blob")
2244        .and_then(Value::as_str)
2245        .ok_or_else(invalid_payload_envelope)?;
2246    Ok(Some((codec, blob)))
2247}
2248
2249fn invalid_payload_envelope() -> Error {
2250    Error::Codec(
2251        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2252            .to_string(),
2253    )
2254}
2255
2256fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2257    for command in commands {
2258        let Some(command) = command.as_object() else {
2259            continue;
2260        };
2261        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2262            continue;
2263        };
2264        let Some(payload_field) = workflow_command_payload_field(command_type) else {
2265            continue;
2266        };
2267
2268        if let Some(codec) = command.get("payload_codec") {
2269            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2270            validate_payload_codec(codec)?;
2271        }
2272
2273        let payload = command
2274            .get(payload_field)
2275            .ok_or_else(invalid_payload_envelope)?;
2276        if runtime_payloads::Reference::parse(payload)?.is_none() {
2277            validate_outbound_payload_envelope(payload)?;
2278        }
2279    }
2280    Ok(())
2281}
2282
2283fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2284    if commands.iter().any(|command| {
2285        command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2286            && command
2287                .get("condition_wait_occurrence_id")
2288                .and_then(Value::as_str)
2289                .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2290    }) {
2291        CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2292    } else if commands.iter().any(|command| {
2293        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2294            && command.get("attribute_types").is_some()
2295    }) {
2296        TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2297    } else if commands
2298        .iter()
2299        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2300    {
2301        MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2302    } else if commands
2303        .iter()
2304        .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2305    {
2306        CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2307    } else if commands.iter().any(|command| {
2308        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2309    }) {
2310        SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2311    } else {
2312        WORKER_PROTOCOL_VERSION
2313    }
2314}
2315
2316fn workflow_completion_protocol_version_with_message_streams(
2317    commands: &[Value],
2318    has_message_stream_metadata: bool,
2319) -> &'static str {
2320    let command_protocol = workflow_completion_protocol_version(commands);
2321    if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2322        MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2323    } else {
2324        command_protocol
2325    }
2326}
2327
2328fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2329    match command_type {
2330        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2331        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2332        "start_service_operation" => Some("request_payload"),
2333        "upsert_memo" => Some("entries"),
2334        _ => None,
2335    }
2336}
2337
2338fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2339    let Some((codec, blob)) = payload_envelope_parts(value)? else {
2340        return Err(untagged_payload_value());
2341    };
2342    validate_payload_codec(codec)?;
2343    decode_avro_value_blob(blob)?;
2344    Ok(())
2345}
2346
2347fn unsupported_payload_codec(codec: &str) -> Error {
2348    Error::Codec(format!(
2349        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2350    ))
2351}
2352
2353fn untagged_payload_value() -> Error {
2354    Error::Codec(
2355        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2356            .to_string(),
2357    )
2358}
2359
2360fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2361    let bytes = BASE64.decode(blob).map_err(|err| {
2362        Error::Codec(format!(
2363            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2364        ))
2365    })?;
2366
2367    if serde_json::from_slice::<Value>(&bytes).is_ok() {
2368        return Err(unsupported_payload_codec("json"));
2369    }
2370
2371    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2372        return Err(Error::Codec(
2373            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2374        ));
2375    }
2376
2377    let fingerprint: [u8; 8] = bytes[2..10]
2378        .try_into()
2379        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2380    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2381        return Err(Error::Codec(format!(
2382            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2383            fingerprint
2384                .iter()
2385                .map(|byte| format!("{byte:02x}"))
2386                .collect::<String>()
2387        )));
2388    }
2389
2390    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2391    // The current fingerprint selects the current immutable schema, so reader
2392    // resolution would only re-walk the same recursive union. Future retained
2393    // writer fingerprints supply a distinct reader schema in this branch.
2394    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2395    if datum_reader.truncated {
2396        return Err(Error::Codec(
2397            "invalid_payload_framing: truncated Avro Value datum".to_string(),
2398        ));
2399    }
2400    let datum = datum.map_err(|err| {
2401        Error::Codec(format!(
2402            "invalid_payload_framing: malformed Avro Value datum: {err}"
2403        ))
2404    })?;
2405    if datum_reader.remaining() != 0 {
2406        return Err(Error::Codec(format!(
2407            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2408            datum_reader.remaining()
2409        )));
2410    }
2411    avro_value_from_datum(datum)
2412}
2413
2414struct StrictAvroDatumReader<'a> {
2415    bytes: &'a [u8],
2416    offset: usize,
2417    truncated: bool,
2418}
2419
2420impl<'a> StrictAvroDatumReader<'a> {
2421    fn new(bytes: &'a [u8]) -> Self {
2422        Self {
2423            bytes,
2424            offset: 0,
2425            truncated: false,
2426        }
2427    }
2428
2429    fn remaining(&self) -> usize {
2430        self.bytes.len() - self.offset
2431    }
2432}
2433
2434impl Read for StrictAvroDatumReader<'_> {
2435    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2436        let count = buffer.len().min(self.remaining());
2437        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2438        self.offset += count;
2439        if count < buffer.len() {
2440            self.truncated = true;
2441        }
2442
2443        Ok(count)
2444    }
2445}
2446
2447fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2448    let branch = match value {
2449        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2450        AvroValue::Boolean(value) => AvroDatum::Union(
2451            1,
2452            Box::new(AvroDatum::Record(vec![(
2453                "boolean".to_string(),
2454                AvroDatum::Boolean(*value),
2455            )])),
2456        ),
2457        AvroValue::Long(value) => AvroDatum::Union(
2458            2,
2459            Box::new(AvroDatum::Record(vec![(
2460                "long".to_string(),
2461                AvroDatum::Long(*value),
2462            )])),
2463        ),
2464        AvroValue::Double(value) => {
2465            if !value.is_finite() {
2466                return Err(Error::Codec(
2467                    "non_finite_float: Avro Value doubles must be finite".to_string(),
2468                ));
2469            }
2470            AvroDatum::Union(
2471                3,
2472                Box::new(AvroDatum::Record(vec![(
2473                    "double".to_string(),
2474                    AvroDatum::Double(*value),
2475                )])),
2476            )
2477        }
2478        AvroValue::Bytes(value) => AvroDatum::Union(
2479            4,
2480            Box::new(AvroDatum::Record(vec![(
2481                "bytes".to_string(),
2482                AvroDatum::Bytes(value.clone()),
2483            )])),
2484        ),
2485        AvroValue::String(value) => AvroDatum::Union(
2486            5,
2487            Box::new(AvroDatum::Record(vec![(
2488                "string".to_string(),
2489                AvroDatum::String(value.clone()),
2490            )])),
2491        ),
2492        AvroValue::Array(values) => AvroDatum::Union(
2493            6,
2494            Box::new(AvroDatum::Record(vec![(
2495                "items".to_string(),
2496                AvroDatum::Array(
2497                    values
2498                        .iter()
2499                        .map(avro_value_to_datum)
2500                        .collect::<Result<Vec<_>>>()?,
2501                ),
2502            )])),
2503        ),
2504        AvroValue::Map(values) => AvroDatum::Union(
2505            7,
2506            Box::new(AvroDatum::Record(vec![(
2507                "entries".to_string(),
2508                AvroDatum::Array(
2509                    values
2510                        .iter()
2511                        .map(|(key, value)| {
2512                            Ok(AvroDatum::Record(vec![
2513                                ("key".to_string(), AvroDatum::String(key.clone())),
2514                                ("value".to_string(), avro_value_to_datum(value)?),
2515                            ]))
2516                        })
2517                        .collect::<Result<Vec<_>>>()?,
2518                ),
2519            )])),
2520        ),
2521    };
2522    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2523}
2524
2525fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2526    let AvroDatum::Record(mut outer) = datum else {
2527        return Err(Error::Codec(
2528            "invalid_payload_framing: datum is not a Value record".to_string(),
2529        ));
2530    };
2531    let (_, branch) = outer
2532        .pop()
2533        .filter(|(name, _)| name == "value")
2534        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2535    let AvroDatum::Union(_, branch) = branch else {
2536        return Err(Error::Codec(
2537            "invalid_payload_framing: invalid Value union".to_string(),
2538        ));
2539    };
2540    match *branch {
2541        AvroDatum::Null => Ok(AvroValue::Null),
2542        AvroDatum::Record(mut fields) => {
2543            let (name, value) = fields.pop().ok_or_else(|| {
2544                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2545            })?;
2546            match (name.as_str(), value) {
2547                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2548                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2549                ("double", AvroDatum::Double(value)) if value.is_finite() => {
2550                    Ok(AvroValue::Double(value))
2551                }
2552                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2553                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2554                ("items", AvroDatum::Array(values)) => values
2555                    .into_iter()
2556                    .map(avro_value_from_datum)
2557                    .collect::<Result<Vec<_>>>()
2558                    .map(AvroValue::Array),
2559                ("entries", AvroDatum::Map(values)) => values
2560                    .into_iter()
2561                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2562                    .collect::<Result<BTreeMap<_, _>>>()
2563                    .map(AvroValue::Map),
2564                _ => Err(Error::Codec(
2565                    "invalid_payload_framing: unknown Value branch".to_string(),
2566                )),
2567            }
2568        }
2569        _ => Err(Error::Codec(
2570            "invalid_payload_framing: invalid Value branch".to_string(),
2571        )),
2572    }
2573}
2574
2575fn avro_value_schema() -> Result<&'static Schema> {
2576    match AVRO_VALUE_SCHEMA.get_or_init(|| {
2577        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2578            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2579    }) {
2580        Ok(schema) => Ok(schema),
2581        Err(message) => Err(Error::Codec(message.clone())),
2582    }
2583}
2584
2585fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2586    match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2587        // Apache Avro's Value::Map uses a randomized HashMap. Arrays and maps
2588        // have the same block representation when each ordered array record
2589        // contains the map key followed by its value, so this encoding-only
2590        // adaptation lets the official encoder retain BTreeMap wire order.
2591        let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2592            .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2593        let entries_schema = schema
2594            .pointer_mut("/fields/0/type/7/fields/0/type")
2595            .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2596        if *entries_schema != json!({"type": "map", "values": "Value"}) {
2597            return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2598        }
2599        *entries_schema = json!({
2600            "type": "array",
2601            "items": {
2602                "type": "record",
2603                "name": "MapEntry",
2604                "fields": [
2605                    {"name": "key", "type": "string"},
2606                    {"name": "value", "type": "Value"}
2607                ]
2608            }
2609        });
2610        Schema::parse_str(&schema.to_string())
2611            .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2612    }) {
2613        Ok(schema) => Ok(schema),
2614        Err(message) => Err(Error::Codec(message.clone())),
2615    }
2616}
2617
2618#[derive(Clone, Debug)]
2619pub struct Client {
2620    http: reqwest::Client,
2621    base_url: String,
2622    token: Option<String>,
2623    control_token: Option<String>,
2624    worker_token: Option<String>,
2625    namespace: String,
2626    max_external_payload_bytes: usize,
2627    worker_storage_admission: Option<WorkerStorageAdmission>,
2628    runtime_upload_policy: Arc<Mutex<runtime_uploads::PolicyCache>>,
2629}
2630
2631impl Client {
2632    pub fn new(base_url: impl Into<String>) -> Result<Self> {
2633        Self::builder(base_url).build()
2634    }
2635
2636    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2637        ClientBuilder {
2638            base_url: base_url.into(),
2639            token: None,
2640            control_token: None,
2641            worker_token: None,
2642            namespace: "default".to_string(),
2643            timeout: Duration::from_secs(60),
2644            max_external_payload_bytes: 64 * 1024 * 1024,
2645        }
2646    }
2647
2648    pub async fn health(&self) -> Result<Value> {
2649        self.request_json(
2650            reqwest::Method::GET,
2651            "/health",
2652            RequestProtocol::ControlPlane,
2653            Option::<&Value>::None,
2654        )
2655        .await
2656    }
2657
2658    pub async fn cluster_info(&self) -> Result<Value> {
2659        self.request_json(
2660            reqwest::Method::GET,
2661            "/cluster/info",
2662            RequestProtocol::ControlPlane,
2663            Option::<&Value>::None,
2664        )
2665        .await
2666    }
2667
2668    pub async fn start_workflow<T: Serialize>(
2669        &self,
2670        workflow_type: &str,
2671        task_queue: &str,
2672        workflow_id: &str,
2673        input: T,
2674    ) -> Result<WorkflowHandle> {
2675        self.start_workflow_with_options(
2676            workflow_type,
2677            task_queue,
2678            workflow_id,
2679            WorkflowStartOptions::default(),
2680            input,
2681        )
2682        .await
2683    }
2684
2685    /// Start a workflow with explicit server-enforced execution and run
2686    /// deadlines.
2687    pub async fn start_workflow_with_options<T: Serialize>(
2688        &self,
2689        workflow_type: &str,
2690        task_queue: &str,
2691        workflow_id: &str,
2692        options: WorkflowStartOptions,
2693        input: T,
2694    ) -> Result<WorkflowHandle> {
2695        options.validate()?;
2696        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2697        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2698        let body = json!({
2699            "workflow_id": workflow_id,
2700            "workflow_type": workflow_type,
2701            "task_queue": task_queue,
2702            "input": input_envelope,
2703            "execution_timeout_seconds": options.execution_timeout_seconds,
2704            "run_timeout_seconds": options.run_timeout_seconds
2705        });
2706
2707        let data: Value = self
2708            .request_json(
2709                reqwest::Method::POST,
2710                "/workflows",
2711                RequestProtocol::ControlPlane,
2712                Some(&body),
2713            )
2714            .await?;
2715
2716        Ok(WorkflowHandle {
2717            client: self.clone(),
2718            workflow_id: data
2719                .get("workflow_id")
2720                .and_then(Value::as_str)
2721                .unwrap_or(workflow_id)
2722                .to_string(),
2723            run_id: data
2724                .get("run_id")
2725                .and_then(Value::as_str)
2726                .map(str::to_string),
2727            workflow_type: data
2728                .get("workflow_type")
2729                .and_then(Value::as_str)
2730                .unwrap_or(workflow_type)
2731                .to_string(),
2732        })
2733    }
2734
2735    pub async fn signal_workflow<T: Serialize>(
2736        &self,
2737        workflow_id: &str,
2738        signal_name: &str,
2739        input: T,
2740    ) -> Result<Value> {
2741        self.signal_workflow_target(workflow_id, None, signal_name, input)
2742            .await
2743    }
2744
2745    /// Append one idempotently identified item to an instance-scoped input stream.
2746    pub async fn append_message_stream<T: Serialize>(
2747        &self,
2748        workflow_id: &str,
2749        stream_name: &str,
2750        message_id: &str,
2751        input: T,
2752    ) -> Result<Value> {
2753        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2754        let body = json!({
2755            "message_id": message_id,
2756            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2757        });
2758        self.request_json(
2759            reqwest::Method::POST,
2760            &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2761            RequestProtocol::ControlPlane,
2762            Some(&body),
2763        )
2764        .await
2765    }
2766
2767    /// Signal only if `run_id` is still the current run for this instance.
2768    pub async fn signal_workflow_run<T: Serialize>(
2769        &self,
2770        workflow_id: &str,
2771        run_id: &str,
2772        signal_name: &str,
2773        input: T,
2774    ) -> Result<Value> {
2775        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2776            .await
2777    }
2778
2779    async fn signal_workflow_target<T: Serialize>(
2780        &self,
2781        workflow_id: &str,
2782        run_id: Option<&str>,
2783        signal_name: &str,
2784        input: T,
2785    ) -> Result<Value> {
2786        validate_user_signal_name(signal_name)?;
2787        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2788        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2789        let body = json!({
2790            "input": input_envelope
2791        });
2792        let path = match run_id {
2793            Some(run_id) => {
2794                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2795            }
2796            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2797        };
2798        self.request_json(
2799            reqwest::Method::POST,
2800            &path,
2801            RequestProtocol::ControlPlane,
2802            Some(&body),
2803        )
2804        .await
2805    }
2806
2807    /// Request cooperative cancellation of the current run for an instance.
2808    pub async fn cancel_workflow(
2809        &self,
2810        workflow_id: &str,
2811        options: WorkflowCommandOptions,
2812    ) -> Result<WorkflowCommandResult> {
2813        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2814            .await
2815    }
2816
2817    /// Request cooperative cancellation only if `run_id` is still current.
2818    pub async fn cancel_workflow_run(
2819        &self,
2820        workflow_id: &str,
2821        run_id: &str,
2822        options: WorkflowCommandOptions,
2823    ) -> Result<WorkflowCommandResult> {
2824        self.workflow_command(
2825            workflow_id,
2826            Some(run_id),
2827            WorkflowCommandKind::Cancel,
2828            options,
2829        )
2830        .await
2831    }
2832
2833    /// Forcefully terminate the current run for an instance.
2834    pub async fn terminate_workflow(
2835        &self,
2836        workflow_id: &str,
2837        options: WorkflowCommandOptions,
2838    ) -> Result<WorkflowCommandResult> {
2839        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2840            .await
2841    }
2842
2843    /// Forcefully terminate only if `run_id` is still current.
2844    pub async fn terminate_workflow_run(
2845        &self,
2846        workflow_id: &str,
2847        run_id: &str,
2848        options: WorkflowCommandOptions,
2849    ) -> Result<WorkflowCommandResult> {
2850        self.workflow_command(
2851            workflow_id,
2852            Some(run_id),
2853            WorkflowCommandKind::Terminate,
2854            options,
2855        )
2856        .await
2857    }
2858
2859    async fn workflow_command(
2860        &self,
2861        workflow_id: &str,
2862        run_id: Option<&str>,
2863        command: WorkflowCommandKind,
2864        options: WorkflowCommandOptions,
2865    ) -> Result<WorkflowCommandResult> {
2866        let path = match run_id {
2867            Some(run_id) => format!(
2868                "/workflows/{workflow_id}/runs/{run_id}/{}",
2869                command.as_str()
2870            ),
2871            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2872        };
2873        let data = match self
2874            .request_json(
2875                reqwest::Method::POST,
2876                &path,
2877                RequestProtocol::ControlPlane,
2878                Some(&options),
2879            )
2880            .await
2881        {
2882            Ok(data) => data,
2883            Err(Error::Http { status, body }) => {
2884                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2885                    command,
2886                    status,
2887                    body,
2888                    workflow_id,
2889                    run_id,
2890                )));
2891            }
2892            Err(error) => return Err(error),
2893        };
2894
2895        Ok(workflow_command_result(command, data, workflow_id, run_id))
2896    }
2897
2898    /// Execute a named, read-only query against a running or completed workflow.
2899    ///
2900    /// Arguments and results use the platform payload envelope. Server and
2901    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
2902    /// reason, HTTP status, and original response body.
2903    pub async fn query_workflow<T: Serialize>(
2904        &self,
2905        workflow_id: &str,
2906        query_name: &str,
2907        input: T,
2908    ) -> Result<Value> {
2909        self.query_workflow_target(workflow_id, None, query_name, input)
2910            .await
2911    }
2912
2913    /// Query only if `run_id` is still current, preventing accidental retargeting.
2914    pub async fn query_workflow_run<T: Serialize>(
2915        &self,
2916        workflow_id: &str,
2917        run_id: &str,
2918        query_name: &str,
2919        input: T,
2920    ) -> Result<Value> {
2921        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2922            .await
2923    }
2924
2925    /// Query a workflow and return the lossless fixed Avro Value result.
2926    pub async fn query_workflow_avro_value<T: Serialize>(
2927        &self,
2928        workflow_id: &str,
2929        query_name: &str,
2930        input: T,
2931    ) -> Result<AvroValue> {
2932        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2933            .await
2934    }
2935
2936    /// Query a selected run and return the lossless fixed Avro Value result.
2937    pub async fn query_workflow_run_avro_value<T: Serialize>(
2938        &self,
2939        workflow_id: &str,
2940        run_id: &str,
2941        query_name: &str,
2942        input: T,
2943    ) -> Result<AvroValue> {
2944        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2945            .await
2946    }
2947
2948    async fn query_workflow_avro_value_target<T: Serialize>(
2949        &self,
2950        workflow_id: &str,
2951        run_id: Option<&str>,
2952        query_name: &str,
2953        input: T,
2954    ) -> Result<AvroValue> {
2955        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2956        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2957        let path = match run_id {
2958            Some(run_id) => {
2959                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2960            }
2961            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2962        };
2963        let response: Value = match self
2964            .request_json(
2965                reqwest::Method::POST,
2966                &path,
2967                RequestProtocol::ControlPlane,
2968                Some(&body),
2969            )
2970            .await
2971        {
2972            Ok(response) => response,
2973            Err(Error::Http { status, body }) => {
2974                return Err(Error::QueryFailed(query_failure(status, body)));
2975            }
2976            Err(error) => return Err(error),
2977        };
2978
2979        let envelope = response
2980            .get("result_envelope")
2981            .filter(|envelope| !envelope.is_null())
2982            .ok_or_else(|| {
2983                Error::Codec(
2984                    "missing_payload_envelope: typed query result requires result_envelope"
2985                        .to_string(),
2986                )
2987            })?;
2988        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2989    }
2990
2991    async fn query_workflow_target<T: Serialize>(
2992        &self,
2993        workflow_id: &str,
2994        run_id: Option<&str>,
2995        query_name: &str,
2996        input: T,
2997    ) -> Result<Value> {
2998        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2999        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
3000        let body = json!({
3001            "input": input_envelope
3002        });
3003        let path = match run_id {
3004            Some(run_id) => {
3005                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3006            }
3007            None => format!("/workflows/{workflow_id}/query/{query_name}"),
3008        };
3009        let response: Value = match self
3010            .request_json(
3011                reqwest::Method::POST,
3012                &path,
3013                RequestProtocol::ControlPlane,
3014                Some(&body),
3015            )
3016            .await
3017        {
3018            Ok(response) => response,
3019            Err(Error::Http { status, body }) => {
3020                return Err(Error::QueryFailed(query_failure(status, body)));
3021            }
3022            Err(error) => return Err(error),
3023        };
3024
3025        if let Some(envelope) = response
3026            .get("result_envelope")
3027            .filter(|envelope| !envelope.is_null())
3028        {
3029            return decode_wire_value(envelope, DEFAULT_CODEC);
3030        }
3031
3032        Ok(response.get("result").cloned().unwrap_or(Value::Null))
3033    }
3034
3035    /// Send a synchronous update using fixed Avro Value arguments.
3036    pub async fn update_workflow<T: Serialize>(
3037        &self,
3038        workflow_id: &str,
3039        update_name: &str,
3040        input: T,
3041        request_id: Option<&str>,
3042    ) -> Result<Value> {
3043        let response = self
3044            .update_workflow_response(workflow_id, update_name, input, request_id)
3045            .await?;
3046        if let Some(envelope) = response
3047            .get("result_envelope")
3048            .filter(|envelope| !envelope.is_null())
3049        {
3050            return decode_wire_value(envelope, DEFAULT_CODEC);
3051        }
3052        Ok(response.get("result").cloned().unwrap_or(response))
3053    }
3054
3055    /// Send a synchronous update and retain a bytes-capable Avro result.
3056    pub async fn update_workflow_avro_value<T: Serialize>(
3057        &self,
3058        workflow_id: &str,
3059        update_name: &str,
3060        input: T,
3061        request_id: Option<&str>,
3062    ) -> Result<AvroValue> {
3063        let response = self
3064            .update_workflow_response(workflow_id, update_name, input, request_id)
3065            .await?;
3066        let envelope = response
3067            .get("result_envelope")
3068            .filter(|envelope| !envelope.is_null())
3069            .ok_or_else(|| {
3070                Error::Codec(
3071                    "missing_payload_envelope: typed update result requires result_envelope"
3072                        .to_string(),
3073                )
3074            })?;
3075        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3076    }
3077
3078    async fn update_workflow_response<T: Serialize>(
3079        &self,
3080        workflow_id: &str,
3081        update_name: &str,
3082        input: T,
3083        request_id: Option<&str>,
3084    ) -> Result<Value> {
3085        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3086        let mut body = json!({
3087            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3088            "wait_for": "completed",
3089        });
3090        if let Some(request_id) = request_id {
3091            body["request_id"] = json!(request_id);
3092        }
3093        self.request_json(
3094            reqwest::Method::POST,
3095            &format!("/workflows/{workflow_id}/update/{update_name}"),
3096            RequestProtocol::ControlPlane,
3097            Some(&body),
3098        )
3099        .await
3100    }
3101
3102    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3103        let path = format!("/workflows/{workflow_id}");
3104        let mut data: WorkflowDescription = self
3105            .request_json(
3106                reqwest::Method::GET,
3107                &path,
3108                RequestProtocol::ControlPlane,
3109                Option::<&Value>::None,
3110            )
3111            .await?;
3112        data.decode_payloads()?;
3113        Ok(data)
3114    }
3115
3116    /// Describe one selected run, including historical terminal runs.
3117    pub async fn describe_workflow_run(
3118        &self,
3119        workflow_id: &str,
3120        run_id: &str,
3121    ) -> Result<WorkflowDescription> {
3122        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3123        let mut data: WorkflowDescription = self
3124            .request_json(
3125                reqwest::Method::GET,
3126                &path,
3127                RequestProtocol::ControlPlane,
3128                Option::<&Value>::None,
3129            )
3130            .await?;
3131        data.decode_payloads()?;
3132        Ok(data)
3133    }
3134
3135    fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3136        let mut path = format!(
3137            "/workflows/{}/runs/{}/streams",
3138            percent_encode_path_segment(workflow_id),
3139            percent_encode_path_segment(run_id),
3140        );
3141        if let Some(stream_name) = stream_name {
3142            path.push('/');
3143            path.push_str(&percent_encode_path_segment(stream_name));
3144        }
3145        path
3146    }
3147
3148    /// List the run-scoped output streams already opened by a workflow.
3149    pub async fn list_workflow_streams(
3150        &self,
3151        workflow_id: &str,
3152        run_id: &str,
3153    ) -> Result<Vec<WorkflowStreamDescription>> {
3154        let response: WorkflowStreamListResponse = self
3155            .request_json(
3156                reqwest::Method::GET,
3157                &Self::workflow_stream_path(workflow_id, run_id, None),
3158                RequestProtocol::ControlPlane,
3159                Option::<&Value>::None,
3160            )
3161            .await?;
3162        Ok(response.streams)
3163    }
3164
3165    /// Describe stream lifecycle, offsets, pending count, and terminal error.
3166    pub async fn describe_workflow_stream(
3167        &self,
3168        workflow_id: &str,
3169        run_id: &str,
3170        stream_name: &str,
3171    ) -> Result<WorkflowStreamDescription> {
3172        let response: WorkflowStreamDescriptionResponse = self
3173            .request_json(
3174                reqwest::Method::GET,
3175                &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3176                RequestProtocol::ControlPlane,
3177                Option::<&Value>::None,
3178            )
3179            .await?;
3180        Ok(response.stream)
3181    }
3182
3183    /// Read one bounded page beginning at a zero-based offset.
3184    ///
3185    /// Delivery is at least once: persist `next_offset` only after processing
3186    /// the page. The future is cancellation-safe; dropping it cancels the
3187    /// in-flight request. Long polling is capped at 60 seconds by the SDK and
3188    /// service contract.
3189    pub async fn subscribe_workflow_stream(
3190        &self,
3191        workflow_id: &str,
3192        run_id: &str,
3193        stream_name: &str,
3194        from_offset: u64,
3195        max_items: usize,
3196        wait: Duration,
3197    ) -> Result<WorkflowStreamPage> {
3198        let max_items = max_items.clamp(1, 500);
3199        let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3200        let path = format!(
3201            "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3202            Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3203        );
3204        let response: WorkflowStreamPageResponse = self
3205            .request_json_with_timeout(
3206                reqwest::Method::GET,
3207                &path,
3208                RequestProtocol::ControlPlane,
3209                Option::<&Value>::None,
3210                Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3211            )
3212            .await?;
3213
3214        let items = response
3215            .items
3216            .into_iter()
3217            .map(|raw| {
3218                let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3219                let envelope = raw.get("payload").cloned();
3220                let payload = envelope
3221                    .as_ref()
3222                    .filter(|value| value.get("blob").is_some())
3223                    .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3224                    .transpose()?
3225                    .map(AvroValue::into_json)
3226                    .transpose()?;
3227                Ok(WorkflowStreamItem {
3228                    offset,
3229                    payload,
3230                    payload_envelope: envelope,
3231                    payload_reference: raw
3232                        .get("payload_reference")
3233                        .and_then(Value::as_str)
3234                        .map(str::to_string),
3235                    payload_codec: raw
3236                        .get("payload_codec")
3237                        .and_then(Value::as_str)
3238                        .map(str::to_string),
3239                    idempotency_key: raw
3240                        .get("idempotency_key")
3241                        .and_then(Value::as_str)
3242                        .map(str::to_string),
3243                    item_type: raw
3244                        .get("item_type")
3245                        .and_then(Value::as_str)
3246                        .map(str::to_string),
3247                    content_type: raw
3248                        .get("content_type")
3249                        .and_then(Value::as_str)
3250                        .map(str::to_string),
3251                    origin: raw
3252                        .get("origin")
3253                        .and_then(Value::as_str)
3254                        .map(str::to_string),
3255                    origin_reference: raw
3256                        .get("origin_reference")
3257                        .and_then(Value::as_str)
3258                        .map(str::to_string),
3259                    emitted_at: raw
3260                        .get("emitted_at")
3261                        .and_then(Value::as_str)
3262                        .map(str::to_string),
3263                    raw,
3264                })
3265            })
3266            .collect::<Result<Vec<_>>>()?;
3267        Ok(WorkflowStreamPage {
3268            stream: response.stream,
3269            items,
3270            next_offset: response.next_offset,
3271            terminal: response.terminal,
3272        })
3273    }
3274
3275    /// Append inline Avro envelopes or opaque external payload references.
3276    pub async fn append_workflow_stream(
3277        &self,
3278        workflow_id: &str,
3279        run_id: &str,
3280        stream_name: &str,
3281        items: &[WorkflowStreamAppendItem],
3282        max_pending_items: Option<u64>,
3283    ) -> Result<WorkflowStreamAppendResult> {
3284        if items.is_empty() {
3285            return Err(Error::Codec(
3286                "workflow_stream_items_empty: append requires at least one item".to_string(),
3287            ));
3288        }
3289        let mut body = json!({
3290            "items": items
3291                .iter()
3292                .map(|item| item.wire_value(None))
3293                .collect::<Vec<_>>(),
3294        });
3295        if let Some(max_pending_items) = max_pending_items {
3296            if max_pending_items == 0 {
3297                return Err(Error::Codec(
3298                    "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3299                        .to_string(),
3300                ));
3301            }
3302            body["max_pending_items"] = json!(max_pending_items);
3303        }
3304        let response: WorkflowStreamAppendResponse = self
3305            .request_json(
3306                reqwest::Method::POST,
3307                &format!(
3308                    "{}/items",
3309                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3310                ),
3311                RequestProtocol::ControlPlane,
3312                Some(&body),
3313            )
3314            .await?;
3315        Ok(WorkflowStreamAppendResult {
3316            stream: response.stream,
3317            accepted_offsets: response.accepted_offsets,
3318            accepted: response.accepted,
3319            deduped: response.deduped,
3320        })
3321    }
3322
3323    /// Close a stream, or mark it errored when `error_reason` is supplied.
3324    pub async fn close_workflow_stream(
3325        &self,
3326        workflow_id: &str,
3327        run_id: &str,
3328        stream_name: &str,
3329        error_reason: Option<&str>,
3330        retention_seconds: Option<u64>,
3331    ) -> Result<WorkflowStreamDescription> {
3332        let mut body = json!({});
3333        if let Some(error_reason) = error_reason {
3334            body["error_reason"] = json!(error_reason);
3335        }
3336        if let Some(retention_seconds) = retention_seconds {
3337            if retention_seconds == 0 {
3338                return Err(Error::Codec(
3339                    "workflow_stream_retention_invalid: retention_seconds must be positive"
3340                        .to_string(),
3341                ));
3342            }
3343            body["retention_seconds"] = json!(retention_seconds);
3344        }
3345        let response: WorkflowStreamDescriptionResponse = self
3346            .request_json(
3347                reqwest::Method::POST,
3348                &format!(
3349                    "{}/close",
3350                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3351                ),
3352                RequestProtocol::ControlPlane,
3353                Some(&body),
3354            )
3355            .await?;
3356        Ok(response.stream)
3357    }
3358
3359    pub async fn register_worker(
3360        &self,
3361        worker_id: &str,
3362        task_queue: &str,
3363        supported_workflow_types: Vec<String>,
3364        supported_activity_types: Vec<String>,
3365        max_concurrent_workflow_tasks: usize,
3366        max_concurrent_activity_tasks: usize,
3367    ) -> Result<RegisterWorkerResponse> {
3368        self.register_worker_with_capabilities(
3369            worker_id,
3370            task_queue,
3371            supported_workflow_types,
3372            supported_activity_types,
3373            max_concurrent_workflow_tasks,
3374            max_concurrent_activity_tasks,
3375            Vec::new(),
3376        )
3377        .await
3378    }
3379
3380    /// Register a worker and explicitly advertise additive worker capabilities.
3381    pub async fn register_worker_with_capabilities(
3382        &self,
3383        worker_id: &str,
3384        task_queue: &str,
3385        supported_workflow_types: Vec<String>,
3386        supported_activity_types: Vec<String>,
3387        max_concurrent_workflow_tasks: usize,
3388        max_concurrent_activity_tasks: usize,
3389        capabilities: Vec<String>,
3390    ) -> Result<RegisterWorkerResponse> {
3391        self.register_worker_with_command_contracts(
3392            worker_id,
3393            task_queue,
3394            supported_workflow_types,
3395            supported_activity_types,
3396            max_concurrent_workflow_tasks,
3397            max_concurrent_activity_tasks,
3398            capabilities,
3399            Value::Object(serde_json::Map::new()),
3400        )
3401        .await
3402    }
3403
3404    /// Register a worker and advertise its named query and update handlers.
3405    ///
3406    /// This Rust SDK cannot execute synchronous pre-accept update validation,
3407    /// so a workflow contract with a non-empty or malformed
3408    /// `update_validators` declaration returns
3409    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
3410    #[allow(clippy::too_many_arguments)]
3411    pub async fn register_worker_with_command_contracts(
3412        &self,
3413        worker_id: &str,
3414        task_queue: &str,
3415        supported_workflow_types: Vec<String>,
3416        supported_activity_types: Vec<String>,
3417        max_concurrent_workflow_tasks: usize,
3418        max_concurrent_activity_tasks: usize,
3419        capabilities: Vec<String>,
3420        workflow_command_contracts: Value,
3421    ) -> Result<RegisterWorkerResponse> {
3422        if let Some(contracts) = workflow_command_contracts.as_object() {
3423            for (workflow_type, contract) in contracts {
3424                let Some(update_validators) = contract.get("update_validators") else {
3425                    continue;
3426                };
3427                if !update_validators
3428                    .as_array()
3429                    .is_some_and(|validators| validators.is_empty())
3430                {
3431                    return Err(Error::UnsupportedUpdateValidators {
3432                        workflow_type: workflow_type.clone(),
3433                    });
3434                }
3435            }
3436        }
3437
3438        let mut body = json!({
3439            "worker_id": worker_id,
3440            "task_queue": task_queue,
3441            "runtime": "rust",
3442            "sdk_version": SDK_VERSION,
3443            "supported_workflow_types": supported_workflow_types,
3444            "supported_activity_types": supported_activity_types,
3445            "capabilities": capabilities,
3446            "capability_manifest": portable_worker_affinity_capability_manifest(),
3447            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3448            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3449        });
3450        if workflow_command_contracts
3451            .as_object()
3452            .is_some_and(|contracts| !contracts.is_empty())
3453        {
3454            body["workflow_command_contracts"] = workflow_command_contracts;
3455        }
3456
3457        self.request_json(
3458            reqwest::Method::POST,
3459            "/worker/register",
3460            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3461            Some(&body),
3462        )
3463        .await
3464    }
3465
3466    /// Gracefully remove one worker's registration through the worker plane.
3467    ///
3468    /// This operation is separate from operator-facing worker management. It
3469    /// uses worker-protocol authentication and returns the server's lease
3470    /// recovery result.
3471    pub async fn deregister_worker_registration(
3472        &self,
3473        worker_id: &str,
3474    ) -> Result<WorkerDeregistrationEnvelope> {
3475        let path = format!(
3476            "/worker/registrations/{}",
3477            percent_encode_path_segment(worker_id)
3478        );
3479        self.request_json(
3480            reqwest::Method::DELETE,
3481            &path,
3482            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3483            Option::<&Value>::None,
3484        )
3485        .await
3486    }
3487
3488    /// Long-poll for an ephemeral, read-only workflow query task.
3489    pub async fn poll_query_task(
3490        &self,
3491        worker_id: &str,
3492        task_queue: &str,
3493        timeout: Duration,
3494    ) -> Result<Option<QueryTask>> {
3495        Ok(self
3496            .poll_query_task_response(worker_id, task_queue, timeout)
3497            .await?
3498            .task)
3499    }
3500
3501    /// Poll a query task while preserving server stop and drain metadata.
3502    pub async fn poll_query_task_response(
3503        &self,
3504        worker_id: &str,
3505        task_queue: &str,
3506        timeout: Duration,
3507    ) -> Result<PollQueryTaskResponse> {
3508        let poll_request_id = unique_request_id("rust-query-poll");
3509        self.poll_query_task_response_with_request_id(
3510            worker_id,
3511            task_queue,
3512            timeout,
3513            &poll_request_id,
3514            1,
3515        )
3516        .await
3517    }
3518
3519    async fn poll_query_task_response_with_request_id(
3520        &self,
3521        worker_id: &str,
3522        task_queue: &str,
3523        timeout: Duration,
3524        poll_request_id: &str,
3525        transport_retries: usize,
3526    ) -> Result<PollQueryTaskResponse> {
3527        let timeout_seconds = long_poll_timeout_seconds(timeout);
3528        let body = json!({
3529            "worker_id": worker_id,
3530            "task_queue": task_queue,
3531            "poll_request_id": poll_request_id,
3532            "timeout_seconds": timeout_seconds,
3533        });
3534        self.poll_request_json(
3535            "/worker/query-tasks/poll",
3536            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3537            &body,
3538            timeout + Duration::from_secs(5),
3539            transport_retries,
3540        )
3541        .await
3542    }
3543
3544    /// Complete a query task without appending workflow history.
3545    pub async fn complete_query_task<T: Serialize>(
3546        &self,
3547        query_task_id: &str,
3548        lease_owner: &str,
3549        query_task_attempt: u64,
3550        result: T,
3551        codec: &str,
3552    ) -> Result<Value> {
3553        let typed_result = AvroValue::from_serialize(&result)?;
3554        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3555        self.complete_query_task_with_envelope(
3556            query_task_id,
3557            lease_owner,
3558            query_task_attempt,
3559            typed_result.into_json()?,
3560            result_envelope,
3561        )
3562        .await
3563    }
3564
3565    async fn complete_query_task_with_envelope(
3566        &self,
3567        query_task_id: &str,
3568        lease_owner: &str,
3569        query_task_attempt: u64,
3570        result: Value,
3571        result_envelope: Value,
3572    ) -> Result<Value> {
3573        let body = json!({
3574            "lease_owner": lease_owner,
3575            "query_task_attempt": query_task_attempt,
3576            "result": result,
3577            "result_envelope": result_envelope,
3578        });
3579        let path = format!("/worker/query-tasks/{query_task_id}/complete");
3580        let response = self
3581            .request_json(
3582                reqwest::Method::POST,
3583                &path,
3584                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3585                Some(&body),
3586            )
3587            .await;
3588        query_task_response(response)
3589    }
3590
3591    /// Report a stable machine-readable query-task failure.
3592    pub async fn fail_query_task(
3593        &self,
3594        query_task_id: &str,
3595        lease_owner: &str,
3596        query_task_attempt: u64,
3597        message: impl Into<String>,
3598        reason: impl Into<String>,
3599        failure_type: impl Into<String>,
3600    ) -> Result<Value> {
3601        let body = json!({
3602            "lease_owner": lease_owner,
3603            "query_task_attempt": query_task_attempt,
3604            "failure": {
3605                "message": message.into(),
3606                "reason": reason.into(),
3607                "type": failure_type.into(),
3608            }
3609        });
3610        let path = format!("/worker/query-tasks/{query_task_id}/fail");
3611        let response = self
3612            .request_json(
3613                reqwest::Method::POST,
3614                &path,
3615                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3616                Some(&body),
3617            )
3618            .await;
3619        query_task_response(response)
3620    }
3621
3622    pub async fn heartbeat_worker(
3623        &self,
3624        worker_id: &str,
3625        workflow_available: usize,
3626        activity_available: usize,
3627    ) -> Result<Value> {
3628        let body = json!({
3629            "worker_id": worker_id,
3630            "task_slots": {
3631                "workflow_available": workflow_available,
3632                "activity_available": activity_available
3633            },
3634            "process_metrics": {
3635                "process_id": std::process::id(),
3636                "process_uptime_seconds": 0
3637            }
3638        });
3639
3640        self.request_json(
3641            reqwest::Method::POST,
3642            "/worker/heartbeat",
3643            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3644            Some(&body),
3645        )
3646        .await
3647    }
3648
3649    pub async fn poll_workflow_task(
3650        &self,
3651        worker_id: &str,
3652        task_queue: &str,
3653        timeout: Duration,
3654    ) -> Result<Option<WorkflowTask>> {
3655        Ok(self
3656            .poll_workflow_task_response(worker_id, task_queue, timeout)
3657            .await?
3658            .task)
3659    }
3660
3661    pub async fn poll_workflow_task_response(
3662        &self,
3663        worker_id: &str,
3664        task_queue: &str,
3665        timeout: Duration,
3666    ) -> Result<PollWorkflowTaskResponse> {
3667        let poll_request_id = unique_request_id("rust-workflow-poll");
3668        self.poll_workflow_task_response_with_request_id(
3669            worker_id,
3670            task_queue,
3671            timeout,
3672            &poll_request_id,
3673            1,
3674        )
3675        .await
3676    }
3677
3678    async fn poll_workflow_task_response_with_request_id(
3679        &self,
3680        worker_id: &str,
3681        task_queue: &str,
3682        timeout: Duration,
3683        poll_request_id: &str,
3684        transport_retries: usize,
3685    ) -> Result<PollWorkflowTaskResponse> {
3686        let body = json!({
3687            "worker_id": worker_id,
3688            "task_queue": task_queue,
3689            "poll_request_id": poll_request_id,
3690            "timeout_seconds": long_poll_timeout_seconds(timeout),
3691        });
3692        let mut data: PollWorkflowTaskResponse = self
3693            .poll_request_json(
3694                "/worker/workflow-tasks/poll",
3695                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3696                &body,
3697                timeout + Duration::from_secs(5),
3698                transport_retries,
3699            )
3700            .await?;
3701
3702        if let Some(task) = data.task.as_mut() {
3703            self.fetch_remaining_workflow_history(worker_id, task)
3704                .await?;
3705        }
3706
3707        Ok(data)
3708    }
3709
3710    async fn fetch_remaining_workflow_history(
3711        &self,
3712        worker_id: &str,
3713        task: &mut WorkflowTask,
3714    ) -> Result<()> {
3715        let mut next_token = task.next_history_page_token.clone();
3716
3717        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3718            let lease_owner = task
3719                .lease_owner
3720                .clone()
3721                .unwrap_or_else(|| worker_id.to_string());
3722            let page = self
3723                .workflow_task_history_page(
3724                    &task.task_id,
3725                    &lease_owner,
3726                    task.workflow_task_attempt,
3727                    &token,
3728                )
3729                .await?;
3730
3731            task.append_history_page(page);
3732
3733            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3734                return Err(Error::Codec(
3735                    "workflow history pagination returned the same page token".to_string(),
3736                ));
3737            }
3738
3739            next_token = task.next_history_page_token.clone();
3740        }
3741
3742        Ok(())
3743    }
3744
3745    async fn workflow_task_history_page(
3746        &self,
3747        task_id: &str,
3748        lease_owner: &str,
3749        workflow_task_attempt: u64,
3750        next_history_page_token: &str,
3751    ) -> Result<WorkflowTaskHistoryPage> {
3752        let body = json!({
3753            "lease_owner": lease_owner,
3754            "workflow_task_attempt": workflow_task_attempt,
3755            "next_history_page_token": next_history_page_token
3756        });
3757        let path = format!("/worker/workflow-tasks/{task_id}/history");
3758
3759        self.request_json(
3760            reqwest::Method::POST,
3761            &path,
3762            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3763            Some(&body),
3764        )
3765        .await
3766    }
3767
3768    pub async fn complete_workflow_task(
3769        &self,
3770        task_id: &str,
3771        lease_owner: &str,
3772        workflow_task_attempt: u64,
3773        commands: Vec<Value>,
3774    ) -> Result<Value> {
3775        self.complete_workflow_task_with_message_streams(
3776            task_id,
3777            lease_owner,
3778            workflow_task_attempt,
3779            commands,
3780            Vec::new(),
3781            Vec::new(),
3782        )
3783        .await
3784    }
3785
3786    async fn complete_workflow_task_with_message_streams(
3787        &self,
3788        task_id: &str,
3789        lease_owner: &str,
3790        workflow_task_attempt: u64,
3791        commands: Vec<Value>,
3792        message_stream_cursors: Vec<Value>,
3793        message_stream_waits: Vec<Value>,
3794    ) -> Result<Value> {
3795        validate_workflow_task_commands(&commands)?;
3796        let has_message_stream_metadata =
3797            !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3798        if has_message_stream_metadata
3799            && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3800        {
3801            return Err(Error::Codec(
3802                "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3803                    .to_string(),
3804            ));
3805        }
3806        let protocol_version = workflow_completion_protocol_version_with_message_streams(
3807            &commands,
3808            has_message_stream_metadata,
3809        );
3810        let mut body = json!({
3811            "lease_owner": lease_owner,
3812            "workflow_task_attempt": workflow_task_attempt,
3813            "commands": commands
3814        });
3815        if !message_stream_cursors.is_empty() {
3816            body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3817        }
3818        if !message_stream_waits.is_empty() {
3819            body["message_stream_waits"] = Value::Array(message_stream_waits);
3820        }
3821        let path = format!("/worker/workflow-tasks/{task_id}/complete");
3822        self.request_json(
3823            reqwest::Method::POST,
3824            &path,
3825            RequestProtocol::Worker(protocol_version),
3826            Some(&body),
3827        )
3828        .await
3829    }
3830
3831    pub async fn fail_workflow_task(
3832        &self,
3833        task_id: &str,
3834        lease_owner: &str,
3835        workflow_task_attempt: u64,
3836        message: impl Into<String>,
3837    ) -> Result<Value> {
3838        self.fail_workflow_task_with_type(
3839            task_id,
3840            lease_owner,
3841            workflow_task_attempt,
3842            message,
3843            "RustWorkflowTaskFailure",
3844        )
3845        .await
3846    }
3847
3848    async fn fail_workflow_task_with_type(
3849        &self,
3850        task_id: &str,
3851        lease_owner: &str,
3852        workflow_task_attempt: u64,
3853        message: impl Into<String>,
3854        failure_type: &str,
3855    ) -> Result<Value> {
3856        let body = json!({
3857            "lease_owner": lease_owner,
3858            "workflow_task_attempt": workflow_task_attempt,
3859            "failure": {
3860                "message": message.into(),
3861                "type": failure_type
3862            }
3863        });
3864        let path = format!("/worker/workflow-tasks/{task_id}/fail");
3865        self.request_json(
3866            reqwest::Method::POST,
3867            &path,
3868            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3869            Some(&body),
3870        )
3871        .await
3872    }
3873
3874    pub async fn poll_activity_task(
3875        &self,
3876        worker_id: &str,
3877        task_queue: &str,
3878        timeout: Duration,
3879    ) -> Result<Option<ActivityTask>> {
3880        Ok(self
3881            .poll_activity_task_response(worker_id, task_queue, timeout)
3882            .await?
3883            .task)
3884    }
3885
3886    /// Poll an activity task while preserving server stop and drain metadata.
3887    pub async fn poll_activity_task_response(
3888        &self,
3889        worker_id: &str,
3890        task_queue: &str,
3891        timeout: Duration,
3892    ) -> Result<PollActivityTaskResponse> {
3893        let poll_request_id = unique_request_id("rust-activity-poll");
3894        self.poll_activity_task_response_with_request_id(
3895            worker_id,
3896            task_queue,
3897            timeout,
3898            &poll_request_id,
3899            1,
3900        )
3901        .await
3902    }
3903
3904    async fn poll_activity_task_response_with_request_id(
3905        &self,
3906        worker_id: &str,
3907        task_queue: &str,
3908        timeout: Duration,
3909        poll_request_id: &str,
3910        transport_retries: usize,
3911    ) -> Result<PollActivityTaskResponse> {
3912        let body = json!({
3913            "worker_id": worker_id,
3914            "task_queue": task_queue,
3915            "poll_request_id": poll_request_id,
3916            "timeout_seconds": long_poll_timeout_seconds(timeout),
3917        });
3918        let data: PollActivityTaskResponse = self
3919            .poll_request_json(
3920                "/worker/activity-tasks/poll",
3921                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3922                &body,
3923                timeout + Duration::from_secs(5),
3924                transport_retries,
3925            )
3926            .await?;
3927        Ok(data)
3928    }
3929
3930    pub async fn complete_activity_task<T: Serialize>(
3931        &self,
3932        task_id: &str,
3933        activity_attempt_id: &str,
3934        lease_owner: &str,
3935        result: T,
3936        codec: &str,
3937    ) -> Result<Value> {
3938        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3939        let body = json!({
3940            "activity_attempt_id": activity_attempt_id,
3941            "lease_owner": lease_owner,
3942            "result": result
3943        });
3944        let path = format!("/worker/activity-tasks/{task_id}/complete");
3945        activity_task_response(
3946            self.request_json(
3947                reqwest::Method::POST,
3948                &path,
3949                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3950                Some(&body),
3951            )
3952            .await,
3953            "complete",
3954            task_id,
3955            activity_attempt_id,
3956        )
3957    }
3958
3959    pub async fn fail_activity_task(
3960        &self,
3961        task_id: &str,
3962        activity_attempt_id: &str,
3963        lease_owner: &str,
3964        message: impl Into<String>,
3965        non_retryable: bool,
3966    ) -> Result<Value> {
3967        let body = json!({
3968            "activity_attempt_id": activity_attempt_id,
3969            "lease_owner": lease_owner,
3970            "failure": {
3971                "message": message.into(),
3972                "type": "RustActivityFailure",
3973                "non_retryable": non_retryable
3974            }
3975        });
3976        let path = format!("/worker/activity-tasks/{task_id}/fail");
3977        activity_task_response(
3978            self.request_json(
3979                reqwest::Method::POST,
3980                &path,
3981                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3982                Some(&body),
3983            )
3984            .await,
3985            "fail",
3986            task_id,
3987            activity_attempt_id,
3988        )
3989    }
3990
3991    pub async fn heartbeat_activity_task<T: Serialize>(
3992        &self,
3993        task_id: &str,
3994        activity_attempt_id: &str,
3995        lease_owner: &str,
3996        details: T,
3997    ) -> Result<ActivityHeartbeatResponse> {
3998        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3999        let body = json!({
4000            "activity_attempt_id": activity_attempt_id,
4001            "lease_owner": lease_owner,
4002            "details": details
4003        });
4004        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4005        activity_task_response(
4006            self.request_json(
4007                reqwest::Method::POST,
4008                &path,
4009                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4010                Some(&body),
4011            )
4012            .await,
4013            "heartbeat",
4014            task_id,
4015            activity_attempt_id,
4016        )
4017    }
4018
4019    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4020        &self,
4021        method: reqwest::Method,
4022        path: &str,
4023        protocol: RequestProtocol,
4024        body: Option<&B>,
4025    ) -> Result<T> {
4026        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4027            .await
4028    }
4029
4030    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4031        &self,
4032        method: reqwest::Method,
4033        path: &str,
4034        protocol: RequestProtocol,
4035        body: Option<&B>,
4036        timeout: Duration,
4037    ) -> Result<T> {
4038        let auth_token = self.auth_token(protocol)?;
4039        let mut request = self
4040            .http
4041            .request(method.clone(), format!("{}/api{}", self.base_url, path))
4042            .timeout(timeout)
4043            .header(reqwest::header::ACCEPT, "application/json")
4044            .header(reqwest::header::CONTENT_TYPE, "application/json")
4045            .header("X-Namespace", &self.namespace);
4046
4047        match protocol {
4048            RequestProtocol::Worker(version) => {
4049                request = request.header("X-Durable-Workflow-Protocol-Version", version);
4050            }
4051            RequestProtocol::ControlPlane => {
4052                request = request.header(
4053                    "X-Durable-Workflow-Control-Plane-Version",
4054                    CONTROL_PLANE_VERSION,
4055                );
4056            }
4057        }
4058
4059        if let Some(token) = auth_token {
4060            request = request.bearer_auth(token);
4061        }
4062
4063        if let Some(body) = body {
4064            let mut body = serde_json::to_value(body)?;
4065            if matches!(
4066                method,
4067                reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH
4068            ) {
4069                self.externalize_runtime_payloads(&mut body, path, protocol)
4070                    .await?;
4071            }
4072            request = request.json(&body);
4073        }
4074
4075        let request = request.build()?;
4076        let poll_request_id = path.ends_with("/poll").then(|| {
4077            request
4078                .body()
4079                .and_then(reqwest::Body::as_bytes)
4080                .and_then(|body| serde_json::from_slice::<Value>(body).ok())
4081                .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4082                .unwrap_or_default()
4083        });
4084        let mut storage_retries = 0_usize;
4085
4086        loop {
4087            // Keep the serialized body and lease/poll identity across admission pauses.
4088            let response = self
4089                .http
4090                .execute(request.try_clone().ok_or_else(|| {
4091                    Error::WorkerLoop("worker request body cannot be retried".to_string())
4092                })?)
4093                .await?;
4094            let status = response.status();
4095            let bytes = response.bytes().await?;
4096
4097            if !status.is_success() {
4098                let body = String::from_utf8_lossy(&bytes).to_string();
4099                if let Some(protocol) = protocol_failure(status, &body) {
4100                    return Err(Error::Protocol(protocol));
4101                }
4102                let error = Error::Http { status, body };
4103                if self
4104                    .wait_for_storage_admission(
4105                        &error,
4106                        protocol,
4107                        Some(path),
4108                        poll_request_id.as_deref(),
4109                        &mut storage_retries,
4110                    )
4111                    .await
4112                {
4113                    continue;
4114                }
4115                return Err(error);
4116            }
4117
4118            if bytes.is_empty() {
4119                return Ok(serde_json::from_value(Value::Null)?);
4120            }
4121
4122            let mut value: Value = serde_json::from_slice(&bytes)?;
4123            self.resolve_runtime_payloads(&mut value, path, protocol)
4124                .await?;
4125            return Ok(serde_json::from_value(value)?);
4126        }
4127    }
4128
4129    async fn wait_for_storage_admission(
4130        &self,
4131        error: &Error,
4132        protocol: RequestProtocol,
4133        path: Option<&str>,
4134        poll_request_id: Option<&str>,
4135        retries: &mut usize,
4136    ) -> bool {
4137        let Some(admission) = self
4138            .worker_storage_admission
4139            .as_ref()
4140            .filter(|_| matches!(protocol, RequestProtocol::Worker(_)))
4141        else {
4142            return false;
4143        };
4144        let Some(advertised_delay) = worker_storage_admission_retry_after(error, poll_request_id)
4145            .or_else(|| {
4146                path.and_then(|path| {
4147                    worker_backend_unavailable_retry_after(error, path, poll_request_id)
4148                })
4149            })
4150        else {
4151            return false;
4152        };
4153        *retries = retries.saturating_add(1);
4154        let delay = worker_retry_delay(admission.policy, *retries)
4155            .max(advertised_delay)
4156            .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4157        let deadline = tokio::time::Instant::now() + delay;
4158        loop {
4159            if admission.stop.load(Ordering::SeqCst) {
4160                return false;
4161            }
4162            let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
4163            if remaining.is_zero() {
4164                return true;
4165            }
4166            tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4167        }
4168    }
4169
4170    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4171        &self,
4172        path: &str,
4173        protocol: RequestProtocol,
4174        body: &B,
4175        timeout: Duration,
4176        max_retries: usize,
4177    ) -> Result<T> {
4178        let mut retries = 0;
4179
4180        loop {
4181            let response = self
4182                .request_json_with_timeout(
4183                    reqwest::Method::POST,
4184                    path,
4185                    protocol,
4186                    Some(body),
4187                    timeout,
4188                )
4189                .await;
4190
4191            match response {
4192                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4193                response => return worker_poll_response(response),
4194            }
4195        }
4196    }
4197
4198    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4199        match protocol {
4200            RequestProtocol::Worker(_) => {
4201                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4202                    return Ok(Some(token));
4203                }
4204                if self.control_token.is_some() {
4205                    return Err(Error::MissingRoleCredentials {
4206                        role: "worker",
4207                        opposite_role: "control",
4208                    });
4209                }
4210                Ok(None)
4211            }
4212            RequestProtocol::ControlPlane => {
4213                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4214                    return Ok(Some(token));
4215                }
4216                if self.worker_token.is_some() {
4217                    return Err(Error::MissingRoleCredentials {
4218                        role: "control",
4219                        opposite_role: "worker",
4220                    });
4221                }
4222                Ok(None)
4223            }
4224        }
4225    }
4226}
4227
4228fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4229    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4230    let reason = body
4231        .get("reason")
4232        .and_then(Value::as_str)
4233        .unwrap_or("query_rejected")
4234        .to_string();
4235    let message = body
4236        .get("message")
4237        .or_else(|| body.get("error"))
4238        .and_then(Value::as_str)
4239        .unwrap_or("workflow query was rejected")
4240        .to_string();
4241
4242    QueryFailure {
4243        status: status.as_u16(),
4244        reason,
4245        message,
4246        body,
4247    }
4248}
4249
4250fn workflow_command_result(
4251    command: WorkflowCommandKind,
4252    data: Value,
4253    workflow_id: &str,
4254    run_id: Option<&str>,
4255) -> WorkflowCommandResult {
4256    WorkflowCommandResult {
4257        command,
4258        workflow_id: data
4259            .get("workflow_id")
4260            .and_then(Value::as_str)
4261            .unwrap_or(workflow_id)
4262            .to_string(),
4263        run_id: data
4264            .get("run_id")
4265            .and_then(Value::as_str)
4266            .or(run_id)
4267            .map(str::to_string),
4268        outcome: data
4269            .get("outcome")
4270            .and_then(Value::as_str)
4271            .map(str::to_string),
4272        reason: data
4273            .get("reason")
4274            .and_then(Value::as_str)
4275            .map(str::to_string),
4276        command_status: data
4277            .get("command_status")
4278            .and_then(Value::as_str)
4279            .map(str::to_string),
4280        raw: data,
4281    }
4282}
4283
4284fn workflow_command_rejection(
4285    command: WorkflowCommandKind,
4286    status: reqwest::StatusCode,
4287    raw_body: String,
4288    workflow_id: &str,
4289    run_id: Option<&str>,
4290) -> WorkflowCommandRejection {
4291    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4292    WorkflowCommandRejection {
4293        command,
4294        status: status.as_u16(),
4295        reason: body
4296            .get("reason")
4297            .and_then(Value::as_str)
4298            .unwrap_or("workflow_command_rejected")
4299            .to_string(),
4300        message: body
4301            .get("message")
4302            .or_else(|| body.get("error"))
4303            .and_then(Value::as_str)
4304            .unwrap_or("workflow lifecycle command was rejected")
4305            .to_string(),
4306        workflow_id: body
4307            .get("workflow_id")
4308            .and_then(Value::as_str)
4309            .unwrap_or(workflow_id)
4310            .to_string(),
4311        run_id: body
4312            .get("run_id")
4313            .and_then(Value::as_str)
4314            .or(run_id)
4315            .map(str::to_string),
4316        target_scope: body
4317            .get("target_scope")
4318            .and_then(Value::as_str)
4319            .map(str::to_string),
4320        body,
4321    }
4322}
4323
4324fn query_task_response(response: Result<Value>) -> Result<Value> {
4325    match response {
4326        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4327        response => response,
4328    }
4329}
4330
4331fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4332    match response {
4333        Err(Error::Http { status, body })
4334            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4335        {
4336            Ok(serde_json::from_str(&body)?)
4337        }
4338        response => response,
4339    }
4340}
4341
4342fn worker_poll_body_is_stop(body: &str) -> bool {
4343    serde_json::from_str::<Value>(body)
4344        .ok()
4345        .is_some_and(|body| {
4346            worker_poll_is_stop(
4347                body.get("poll_status").and_then(Value::as_str),
4348                body.get("reason").and_then(Value::as_str),
4349            )
4350        })
4351}
4352
4353fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4354    matches!(poll_status, Some("draining" | "stopped"))
4355        || matches!(reason, Some("worker_draining" | "worker_stopped"))
4356}
4357
4358fn query_task_rejection_is_final(error: &Error) -> bool {
4359    matches!(
4360        error,
4361        Error::QueryFailed(failure)
4362            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4363    )
4364}
4365
4366fn activity_task_response<T>(
4367    response: Result<T>,
4368    operation: &str,
4369    task_id: &str,
4370    activity_attempt_id: &str,
4371) -> Result<T> {
4372    match response {
4373        Err(Error::Http { status, body }) => {
4374            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4375            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4376                operation: operation.to_string(),
4377                status: status.as_u16(),
4378                reason: body
4379                    .get("reason")
4380                    .and_then(Value::as_str)
4381                    .unwrap_or("activity_task_rejected")
4382                    .to_string(),
4383                task_id: body
4384                    .get("task_id")
4385                    .and_then(Value::as_str)
4386                    .unwrap_or(task_id)
4387                    .to_string(),
4388                activity_attempt_id: body
4389                    .get("activity_attempt_id")
4390                    .and_then(Value::as_str)
4391                    .unwrap_or(activity_attempt_id)
4392                    .to_string(),
4393                cancel_requested: body
4394                    .get("cancel_requested")
4395                    .and_then(Value::as_bool)
4396                    .unwrap_or(false),
4397                can_continue: body.get("can_continue").and_then(Value::as_bool),
4398                run_closed_reason: body
4399                    .get("run_closed_reason")
4400                    .and_then(Value::as_str)
4401                    .map(str::to_string),
4402                body,
4403            }))
4404        }
4405        response => response,
4406    }
4407}
4408
4409fn activity_task_rejection_is_final(error: &Error) -> bool {
4410    matches!(
4411        error,
4412        Error::ActivityTaskRejected(rejection)
4413            if matches!(
4414                rejection.reason.as_str(),
4415                "run_cancelled"
4416                    | "run_terminated"
4417                    | "attempt_closed"
4418                    | "stale_attempt"
4419                    | "activity_cancelled"
4420                    | "task_cancelled"
4421                    | "run_closed"
4422                    | "activity_not_running"
4423                    | "attempt_not_found"
4424            )
4425    )
4426}
4427
4428fn workflow_task_completion_is_terminal_timeout(
4429    error: &Error,
4430    task_id: &str,
4431    workflow_task_attempt: u64,
4432    run_id: Option<&str>,
4433) -> bool {
4434    let Error::Http { status, body } = error else {
4435        return false;
4436    };
4437    if *status != reqwest::StatusCode::CONFLICT {
4438        return false;
4439    }
4440
4441    let Some(run_id) = run_id else {
4442        return false;
4443    };
4444    let Ok(body) = serde_json::from_str::<Value>(body) else {
4445        return false;
4446    };
4447
4448    body.get("recorded").and_then(Value::as_bool) == Some(false)
4449        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4450        && body.get("run_status").and_then(Value::as_str) == Some("failed")
4451        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4452        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4453        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4454}
4455
4456fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4457    let body: Value = serde_json::from_str(raw_body).ok()?;
4458    let reason = body.get("reason")?.as_str()?;
4459    if !matches!(
4460        reason,
4461        "missing_protocol_version"
4462            | "unsupported_protocol_version"
4463            | "missing_control_plane_version"
4464            | "unsupported_control_plane_version"
4465    ) {
4466        return None;
4467    }
4468
4469    Some(ProtocolFailure {
4470        status: status.as_u16(),
4471        reason: reason.to_string(),
4472        message: body
4473            .get("message")
4474            .or_else(|| body.get("error"))
4475            .and_then(Value::as_str)
4476            .unwrap_or("protocol version rejected")
4477            .to_string(),
4478        supported_version: body
4479            .get("supported_version")
4480            .and_then(Value::as_str)
4481            .map(str::to_string),
4482        requested_version: body
4483            .get("requested_version")
4484            .and_then(Value::as_str)
4485            .map(str::to_string),
4486        body,
4487    })
4488}
4489
4490fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4491    timeout
4492        .as_secs()
4493        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4494        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4495}
4496
4497fn worker_operation_is_retryable(error: &Error) -> bool {
4498    if worker_poll_capacity_retry_after(error).is_some()
4499        || worker_storage_admission_body(error).is_some()
4500        || worker_backend_unavailable_body(error).is_some()
4501        || worker_operation_is_explicitly_non_retryable(error)
4502    {
4503        return false;
4504    }
4505
4506    match error {
4507        Error::Transport(error) => {
4508            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4509        }
4510        Error::Http { status, .. } => {
4511            matches!(
4512                *status,
4513                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4514            ) || status.is_server_error()
4515        }
4516        _ => false,
4517    }
4518}
4519
4520fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4521    let body: Value = match error {
4522        Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4523        Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4524        _ => return None,
4525    };
4526    matches!(
4527        body.get("reason").and_then(Value::as_str),
4528        Some("storage_pressure" | "storage_admission_unavailable")
4529    )
4530    .then_some(body)
4531}
4532
4533fn worker_storage_admission_retry_after(
4534    error: &Error,
4535    poll_request_id: Option<&str>,
4536) -> Option<Duration> {
4537    let Error::Http { status, .. } = error else {
4538        return None;
4539    };
4540    let body = worker_storage_admission_body(error)?;
4541    let delay = body.get("retry_after_seconds")?.as_u64()?;
4542    if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4543        || delay == 0
4544        || body.get("retryable") != Some(&Value::Bool(true))
4545        || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4546        || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4547        || body
4548            .get("request_admitted")
4549            .is_some_and(|admitted| admitted != &Value::Bool(false))
4550    {
4551        return None;
4552    }
4553    match poll_request_id {
4554        Some(id) => {
4555            if id.is_empty()
4556                || body.get("task") != Some(&Value::Null)
4557                || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4558                || body.get("poll_status") != body.get("reason")
4559                || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4560                || body.get("claim_admitted") != Some(&Value::Bool(false))
4561            {
4562                return None;
4563            }
4564        }
4565        None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4566        None => {}
4567    }
4568    Some(Duration::from_secs(delay))
4569}
4570
4571fn worker_backend_unavailable_retry_after(
4572    error: &Error,
4573    path: &str,
4574    poll_request_id: Option<&str>,
4575) -> Option<Duration> {
4576    let operation = match path {
4577        "/worker/register" => "register_worker",
4578        "/worker/heartbeat" => "heartbeat_worker",
4579        "/worker/workflow-tasks/poll" => "poll_workflow_task",
4580        "/worker/activity-tasks/poll" => "poll_activity_task",
4581        "/worker/query-tasks/poll" => "poll_query_task",
4582        "/worker/update-validation-tasks/poll" => "poll_update_validation_task",
4583        _ => return None,
4584    };
4585    let body = worker_backend_unavailable_body(error)?;
4586    let delay = body.get("retry_after_seconds")?.as_u64()?;
4587    if delay == 0
4588        || body.get("operation")?.as_str()? != operation
4589        || body.get("outcome")?.as_str()? != "unknown"
4590        || body.get("retryable") != Some(&Value::Bool(true))
4591        || body.get("worker_id")?.as_str()?.is_empty()
4592    {
4593        return None;
4594    }
4595    if let Some(id) = poll_request_id {
4596        if id.is_empty()
4597            || body.get("task") != Some(&Value::Null)
4598            || body.get("poll_status")?.as_str()? != "backend_unavailable"
4599            || body.get("poll_request_id")?.as_str()? != id
4600            || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4601        {
4602            return None;
4603        }
4604    } else if path.ends_with("/poll") {
4605        return None;
4606    }
4607    Some(Duration::from_secs(delay))
4608}
4609
4610fn worker_backend_unavailable_body(error: &Error) -> Option<Value> {
4611    let Error::Http { status, body } = error else {
4612        return None;
4613    };
4614    if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE {
4615        return None;
4616    }
4617    let body: Value = serde_json::from_str(body).ok()?;
4618    (body.get("reason")?.as_str()? == "backend_unavailable").then_some(body)
4619}
4620
4621fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4622    let Error::Http { body, .. } = error else {
4623        return false;
4624    };
4625
4626    serde_json::from_str::<Value>(body)
4627        .ok()
4628        .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4629        == Some(false)
4630}
4631
4632fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4633    let Error::Http { status, body } = error else {
4634        return None;
4635    };
4636    if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4637        return None;
4638    }
4639
4640    let body = serde_json::from_str::<Value>(body).ok()?;
4641    let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4642        == Some("long_poll_capacity_exhausted")
4643        || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4644    if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4645        return None;
4646    }
4647
4648    Some(Duration::from_secs(
4649        body.get("retry_after_seconds")
4650            .and_then(Value::as_u64)
4651            .unwrap_or_default(),
4652    ))
4653}
4654
4655fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4656    let exponent = retry.saturating_sub(1).min(31) as u32;
4657    policy
4658        .initial_backoff
4659        .saturating_mul(1_u32 << exponent)
4660        .min(policy.max_backoff)
4661}
4662
4663#[derive(Debug)]
4664pub struct ClientBuilder {
4665    base_url: String,
4666    token: Option<String>,
4667    control_token: Option<String>,
4668    worker_token: Option<String>,
4669    namespace: String,
4670    timeout: Duration,
4671    max_external_payload_bytes: usize,
4672}
4673
4674impl ClientBuilder {
4675    pub fn token(mut self, token: Option<String>) -> Self {
4676        self.token = token;
4677        self
4678    }
4679
4680    pub fn control_token(mut self, token: Option<String>) -> Self {
4681        self.control_token = token;
4682        self
4683    }
4684
4685    pub fn worker_token(mut self, token: Option<String>) -> Self {
4686        self.worker_token = token;
4687        self
4688    }
4689
4690    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4691        self.namespace = namespace.into();
4692        self
4693    }
4694
4695    pub fn timeout(mut self, timeout: Duration) -> Self {
4696        self.timeout = timeout;
4697        self
4698    }
4699
4700    /// Maximum unique external-payload bytes fetched per response (default 64 MiB).
4701    /// References are fetched only from this client's authenticated runtime.
4702    pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
4703        self.max_external_payload_bytes = bytes;
4704        self
4705    }
4706
4707    pub fn build(self) -> Result<Client> {
4708        let base_url = self.base_url.trim_end_matches('/').to_string();
4709        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4710            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4711            .unwrap_or_else(|_| base_url.ends_with("/api"));
4712
4713        if has_sdk_api_suffix {
4714            return Err(Error::InvalidBaseUrl);
4715        }
4716
4717        Ok(Client {
4718            http: reqwest::Client::builder()
4719                .timeout(self.timeout)
4720                .redirect(reqwest::redirect::Policy::none())
4721                .build()?,
4722            base_url,
4723            token: self.token,
4724            control_token: self.control_token,
4725            worker_token: self.worker_token,
4726            namespace: self.namespace,
4727            max_external_payload_bytes: self.max_external_payload_bytes,
4728            worker_storage_admission: None,
4729            runtime_upload_policy: Arc::new(Mutex::new([None, None])),
4730        })
4731    }
4732}
4733
4734#[derive(Clone, Debug)]
4735pub struct WorkflowHandle {
4736    client: Client,
4737    pub workflow_id: String,
4738    pub run_id: Option<String>,
4739    pub workflow_type: String,
4740}
4741
4742impl WorkflowHandle {
4743    /// Describe whichever run is current for this stable workflow instance.
4744    pub async fn describe(&self) -> Result<WorkflowDescription> {
4745        self.client.describe_workflow(&self.workflow_id).await
4746    }
4747
4748    /// Describe the run identity originally selected by this handle.
4749    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4750        let run_id = self.run_id.as_deref().ok_or_else(|| {
4751            Error::Codec("run_id is required for selected-run description".to_string())
4752        })?;
4753        self.client
4754            .describe_workflow_run(&self.workflow_id, run_id)
4755            .await
4756    }
4757
4758    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4759        self.client
4760            .signal_workflow(&self.workflow_id, signal_name, input)
4761            .await
4762    }
4763
4764    pub async fn append_message<T: Serialize>(
4765        &self,
4766        stream_name: &str,
4767        message_id: &str,
4768        input: T,
4769    ) -> Result<Value> {
4770        self.client
4771            .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4772            .await
4773    }
4774
4775    /// Signal only if this handle's selected run is still current.
4776    pub async fn signal_selected_run<T: Serialize>(
4777        &self,
4778        signal_name: &str,
4779        input: T,
4780    ) -> Result<Value> {
4781        let run_id = self.run_id.as_deref().ok_or_else(|| {
4782            Error::Codec("run_id is required for selected-run signaling".to_string())
4783        })?;
4784        self.client
4785            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4786            .await
4787    }
4788
4789    /// Request cooperative cancellation of whichever run is current.
4790    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4791        self.client
4792            .cancel_workflow(&self.workflow_id, options)
4793            .await
4794    }
4795
4796    /// Request cancellation only if this handle's selected run is still current.
4797    pub async fn cancel_selected_run(
4798        &self,
4799        options: WorkflowCommandOptions,
4800    ) -> Result<WorkflowCommandResult> {
4801        let run_id = self.run_id.as_deref().ok_or_else(|| {
4802            Error::Codec("run_id is required for selected-run cancellation".to_string())
4803        })?;
4804        self.client
4805            .cancel_workflow_run(&self.workflow_id, run_id, options)
4806            .await
4807    }
4808
4809    /// Forcefully terminate whichever run is current.
4810    pub async fn terminate(
4811        &self,
4812        options: WorkflowCommandOptions,
4813    ) -> Result<WorkflowCommandResult> {
4814        self.client
4815            .terminate_workflow(&self.workflow_id, options)
4816            .await
4817    }
4818
4819    /// Terminate only if this handle's selected run is still current.
4820    pub async fn terminate_selected_run(
4821        &self,
4822        options: WorkflowCommandOptions,
4823    ) -> Result<WorkflowCommandResult> {
4824        let run_id = self.run_id.as_deref().ok_or_else(|| {
4825            Error::Codec("run_id is required for selected-run termination".to_string())
4826        })?;
4827        self.client
4828            .terminate_workflow_run(&self.workflow_id, run_id, options)
4829            .await
4830    }
4831
4832    /// Execute a named, read-only query against this workflow.
4833    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4834        self.client
4835            .query_workflow(&self.workflow_id, query_name, input)
4836            .await
4837    }
4838
4839    pub async fn query_avro_value<T: Serialize>(
4840        &self,
4841        query_name: &str,
4842        input: T,
4843    ) -> Result<AvroValue> {
4844        self.client
4845            .query_workflow_avro_value(&self.workflow_id, query_name, input)
4846            .await
4847    }
4848
4849    pub async fn update<T: Serialize>(
4850        &self,
4851        update_name: &str,
4852        input: T,
4853        request_id: Option<&str>,
4854    ) -> Result<Value> {
4855        self.client
4856            .update_workflow(&self.workflow_id, update_name, input, request_id)
4857            .await
4858    }
4859
4860    pub async fn update_avro_value<T: Serialize>(
4861        &self,
4862        update_name: &str,
4863        input: T,
4864        request_id: Option<&str>,
4865    ) -> Result<AvroValue> {
4866        self.client
4867            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4868            .await
4869    }
4870
4871    /// Query only if this handle's selected run is still current.
4872    pub async fn query_selected_run<T: Serialize>(
4873        &self,
4874        query_name: &str,
4875        input: T,
4876    ) -> Result<Value> {
4877        let run_id = self
4878            .run_id
4879            .as_deref()
4880            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4881        self.client
4882            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4883            .await
4884    }
4885
4886    /// Await the final terminal outcome of the current continue-as-new chain.
4887    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4888        self.result_target(options, None).await
4889    }
4890
4891    /// Await the final result without projecting Avro bytes through JSON.
4892    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4893        self.result_avro_value_target(options, None).await
4894    }
4895
4896    /// Await the final result and decode it into a Serde application type.
4897    pub async fn result_typed<T: DeserializeOwned>(
4898        &self,
4899        options: WorkflowResultOptions,
4900    ) -> Result<T> {
4901        let result = self.result_avro_value(options).await?;
4902        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4903    }
4904
4905    /// Await only the run identity originally selected by this handle.
4906    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4907        let run_id = self.run_id.as_deref().ok_or_else(|| {
4908            Error::Codec("run_id is required for selected-run result".to_string())
4909        })?;
4910        self.result_target(options, Some(run_id)).await
4911    }
4912
4913    /// Await the selected run's result on the lossless Avro Value surface.
4914    pub async fn result_selected_run_avro_value(
4915        &self,
4916        options: WorkflowResultOptions,
4917    ) -> Result<AvroValue> {
4918        let run_id = self.run_id.as_deref().ok_or_else(|| {
4919            Error::Codec("run_id is required for selected-run result".to_string())
4920        })?;
4921        self.result_avro_value_target(options, Some(run_id)).await
4922    }
4923
4924    /// Await the selected run and decode its result into a Serde type.
4925    pub async fn result_selected_run_typed<T: DeserializeOwned>(
4926        &self,
4927        options: WorkflowResultOptions,
4928    ) -> Result<T> {
4929        let result = self.result_selected_run_avro_value(options).await?;
4930        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4931    }
4932
4933    async fn result_avro_value_target(
4934        &self,
4935        options: WorkflowResultOptions,
4936        selected_run_id: Option<&str>,
4937    ) -> Result<AvroValue> {
4938        let started = Instant::now();
4939
4940        loop {
4941            let description = match selected_run_id {
4942                Some(run_id) => {
4943                    self.client
4944                        .describe_workflow_run(&self.workflow_id, run_id)
4945                        .await?
4946                }
4947                None => self.describe().await?,
4948            };
4949            if description.is_completed() {
4950                return description.output_avro_value.ok_or_else(|| {
4951                    Error::Codec(
4952                        "missing_payload_envelope: typed workflow result requires output_envelope"
4953                            .to_string(),
4954                    )
4955                });
4956            }
4957            if description.is_terminal() {
4958                let outcome =
4959                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4960                return Err(match outcome.kind {
4961                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4962                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4963                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4964                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4965                });
4966            }
4967            if started.elapsed() >= options.timeout {
4968                return Err(Error::Timeout);
4969            }
4970            tokio::time::sleep(options.poll_interval).await;
4971        }
4972    }
4973
4974    async fn result_target(
4975        &self,
4976        options: WorkflowResultOptions,
4977        selected_run_id: Option<&str>,
4978    ) -> Result<Value> {
4979        let started = Instant::now();
4980
4981        loop {
4982            let description = match selected_run_id {
4983                Some(run_id) => {
4984                    self.client
4985                        .describe_workflow_run(&self.workflow_id, run_id)
4986                        .await?
4987                }
4988                None => self.describe().await?,
4989            };
4990            if description.is_completed() {
4991                return Ok(description.output.unwrap_or(Value::Null));
4992            }
4993
4994            if description.is_terminal() {
4995                let outcome =
4996                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4997                return Err(match outcome.kind {
4998                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4999                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5000                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5001                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5002                });
5003            }
5004
5005            if started.elapsed() >= options.timeout {
5006                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
5007                    kind: WorkflowTerminalKind::TimedOut,
5008                    workflow_id: description
5009                        .workflow_id
5010                        .clone()
5011                        .unwrap_or_else(|| self.workflow_id.clone()),
5012                    run_id: description
5013                        .run_id
5014                        .clone()
5015                        .or_else(|| selected_run_id.map(str::to_string)),
5016                    reason: "result_wait_timeout".to_string(),
5017                    failure_category: Some("client_timeout".to_string()),
5018                    failure_id: None,
5019                    exception_type: None,
5020                    exception_class: None,
5021                    non_retryable: None,
5022                    message: Some(format!(
5023                        "workflow result was not terminal within {:?}",
5024                        options.timeout
5025                    )),
5026                    exception: None,
5027                    raw: description.raw_value(),
5028                }));
5029            }
5030
5031            tokio::time::sleep(options.poll_interval).await;
5032        }
5033    }
5034}
5035
5036#[derive(Clone, Copy, Debug)]
5037pub struct WorkflowResultOptions {
5038    pub poll_interval: Duration,
5039    pub timeout: Duration,
5040}
5041
5042impl Default for WorkflowResultOptions {
5043    fn default() -> Self {
5044        Self {
5045            poll_interval: Duration::from_millis(500),
5046            timeout: Duration::from_secs(30),
5047        }
5048    }
5049}
5050
5051#[derive(Clone, Debug, Deserialize)]
5052pub struct WorkflowDescription {
5053    pub workflow_id: Option<String>,
5054    pub run_id: Option<String>,
5055    pub workflow_type: Option<String>,
5056    pub status: Option<String>,
5057    #[serde(default)]
5058    pub closed_reason: Option<String>,
5059    #[serde(default)]
5060    pub error: Option<String>,
5061    #[serde(default)]
5062    pub failure: Option<Value>,
5063    #[serde(default)]
5064    pub exception: Option<Value>,
5065    #[serde(default)]
5066    pub failures: Vec<Value>,
5067    #[serde(default)]
5068    pub output: Option<Value>,
5069    #[serde(default)]
5070    pub output_envelope: Option<Value>,
5071    #[serde(skip)]
5072    pub output_avro_value: Option<AvroValue>,
5073    #[serde(flatten)]
5074    pub raw: HashMap<String, Value>,
5075}
5076
5077/// Lifecycle and backlog metadata for one run-scoped Workflow Stream.
5078#[derive(Clone, Debug, Deserialize)]
5079pub struct WorkflowStreamDescription {
5080    pub stream_name: String,
5081    pub status: String,
5082    pub last_offset: i64,
5083    pub total_items: u64,
5084    pub pending_items: u64,
5085    #[serde(default)]
5086    pub opened_at: Option<String>,
5087    #[serde(default)]
5088    pub last_appended_at: Option<String>,
5089    #[serde(default)]
5090    pub closed_at: Option<String>,
5091    #[serde(default)]
5092    pub error_reason: Option<String>,
5093    #[serde(default)]
5094    pub retention_seconds: Option<u64>,
5095    #[serde(flatten)]
5096    pub raw: HashMap<String, Value>,
5097}
5098
5099impl WorkflowStreamDescription {
5100    pub fn is_terminal(&self) -> bool {
5101        matches!(self.status.as_str(), "closed" | "errored")
5102    }
5103}
5104
5105/// One item for direct or replay-safe append.
5106#[derive(Clone, Debug, Default)]
5107pub struct WorkflowStreamAppendItem {
5108    pub payload_envelope: Option<Value>,
5109    pub payload_reference: Option<String>,
5110    pub item_type: Option<String>,
5111    pub content_type: Option<String>,
5112    pub idempotency_key: Option<String>,
5113}
5114
5115impl WorkflowStreamAppendItem {
5116    /// Encode an inline payload with the SDK's fixed Avro Value envelope.
5117    pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5118        let value = AvroValue::from_serialize(&payload)?;
5119        Ok(Self {
5120            payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5121            ..Self::default()
5122        })
5123    }
5124
5125    /// Preserve an external payload URI as an opaque service-contract reference.
5126    pub fn from_reference(reference: impl Into<String>) -> Self {
5127        Self {
5128            payload_reference: Some(reference.into()),
5129            ..Self::default()
5130        }
5131    }
5132
5133    pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5134        self.item_type = Some(item_type.into());
5135        self
5136    }
5137
5138    pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5139        self.content_type = Some(content_type.into());
5140        self
5141    }
5142
5143    pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5144        self.idempotency_key = Some(idempotency_key.into());
5145        self
5146    }
5147
5148    fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5149        let mut item = serde_json::Map::new();
5150        if let Some(payload) = &self.payload_envelope {
5151            item.insert("payload".to_string(), payload.clone());
5152            item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5153        }
5154        if let Some(reference) = &self.payload_reference {
5155            item.insert("payload_reference".to_string(), json!(reference));
5156        }
5157        if let Some(item_type) = &self.item_type {
5158            item.insert("item_type".to_string(), json!(item_type));
5159        }
5160        if let Some(content_type) = &self.content_type {
5161            item.insert("content_type".to_string(), json!(content_type));
5162        }
5163        if let Some(key) = derived_idempotency_key
5164            .as_ref()
5165            .or(self.idempotency_key.as_ref())
5166        {
5167            item.insert("idempotency_key".to_string(), json!(key));
5168        }
5169        Value::Object(item)
5170    }
5171}
5172
5173/// One durable item at its stable zero-based offset.
5174#[derive(Clone, Debug)]
5175pub struct WorkflowStreamItem {
5176    pub offset: u64,
5177    pub payload: Option<Value>,
5178    pub payload_envelope: Option<Value>,
5179    pub payload_reference: Option<String>,
5180    pub payload_codec: Option<String>,
5181    pub idempotency_key: Option<String>,
5182    pub item_type: Option<String>,
5183    pub content_type: Option<String>,
5184    pub origin: Option<String>,
5185    pub origin_reference: Option<String>,
5186    pub emitted_at: Option<String>,
5187    pub raw: Value,
5188}
5189
5190/// One bounded at-least-once subscription page.
5191#[derive(Clone, Debug)]
5192pub struct WorkflowStreamPage {
5193    pub stream: WorkflowStreamDescription,
5194    pub items: Vec<WorkflowStreamItem>,
5195    pub next_offset: u64,
5196    pub terminal: bool,
5197}
5198
5199/// Durable acceptance and deduplication outcome for an append request.
5200#[derive(Clone, Debug)]
5201pub struct WorkflowStreamAppendResult {
5202    pub stream: WorkflowStreamDescription,
5203    pub accepted_offsets: Vec<u64>,
5204    pub accepted: u64,
5205    pub deduped: u64,
5206}
5207
5208#[derive(Deserialize)]
5209struct WorkflowStreamListResponse {
5210    #[serde(default)]
5211    streams: Vec<WorkflowStreamDescription>,
5212}
5213
5214#[derive(Deserialize)]
5215struct WorkflowStreamDescriptionResponse {
5216    stream: WorkflowStreamDescription,
5217}
5218
5219#[derive(Deserialize)]
5220struct WorkflowStreamPageResponse {
5221    stream: WorkflowStreamDescription,
5222    #[serde(default)]
5223    items: Vec<Value>,
5224    next_offset: u64,
5225    terminal: bool,
5226}
5227
5228#[derive(Deserialize)]
5229struct WorkflowStreamAppendResponse {
5230    stream: WorkflowStreamDescription,
5231    #[serde(default)]
5232    accepted_offsets: Vec<u64>,
5233    accepted: u64,
5234    deduped: u64,
5235}
5236
5237impl WorkflowDescription {
5238    pub fn is_completed(&self) -> bool {
5239        matches!(self.status.as_deref(), Some("completed" | "Completed"))
5240    }
5241
5242    pub fn is_terminal(&self) -> bool {
5243        matches!(
5244            self.status.as_deref(),
5245            Some(
5246                "completed"
5247                    | "Completed"
5248                    | "failed"
5249                    | "Failed"
5250                    | "cancelled"
5251                    | "Cancelled"
5252                    | "terminated"
5253                    | "Terminated"
5254                    | "timed_out"
5255                    | "TimedOut",
5256            )
5257        )
5258    }
5259
5260    fn decode_payloads(&mut self) -> Result<()> {
5261        if let Some(envelope) = &self.output_envelope {
5262            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5263            self.output = Some(value.clone().into_json()?);
5264            self.output_avro_value = Some(value);
5265        }
5266
5267        Ok(())
5268    }
5269
5270    fn raw_value(&self) -> Value {
5271        let mut data = self.raw.clone();
5272        data.insert(
5273            "workflow_id".to_string(),
5274            self.workflow_id
5275                .clone()
5276                .map(Value::String)
5277                .unwrap_or(Value::Null),
5278        );
5279        data.insert(
5280            "run_id".to_string(),
5281            self.run_id
5282                .clone()
5283                .map(Value::String)
5284                .unwrap_or(Value::Null),
5285        );
5286        data.insert(
5287            "workflow_type".to_string(),
5288            self.workflow_type
5289                .clone()
5290                .map(Value::String)
5291                .unwrap_or(Value::Null),
5292        );
5293        data.insert(
5294            "status".to_string(),
5295            self.status
5296                .clone()
5297                .map(Value::String)
5298                .unwrap_or(Value::Null),
5299        );
5300        data.insert(
5301            "closed_reason".to_string(),
5302            self.closed_reason
5303                .clone()
5304                .map(Value::String)
5305                .unwrap_or(Value::Null),
5306        );
5307        if let Some(failure) = &self.failure {
5308            data.insert("failure".to_string(), failure.clone());
5309        }
5310        if let Some(exception) = &self.exception {
5311            data.insert("exception".to_string(), exception.clone());
5312        }
5313        Value::Object(data.into_iter().collect())
5314    }
5315}
5316
5317fn workflow_terminal_outcome(
5318    description: &WorkflowDescription,
5319    workflow_id: &str,
5320    run_id: Option<&str>,
5321) -> WorkflowTerminalOutcome {
5322    let terminal_kind = description
5323        .closed_reason
5324        .as_deref()
5325        .or(description.status.as_deref())
5326        .unwrap_or("failed")
5327        .to_ascii_lowercase();
5328    let kind = match terminal_kind.as_str() {
5329        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5330        "terminated" => WorkflowTerminalKind::Terminated,
5331        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5332        _ => WorkflowTerminalKind::Failed,
5333    };
5334    let default_reason = match kind {
5335        WorkflowTerminalKind::Failed => "workflow_failed",
5336        WorkflowTerminalKind::Cancelled => "cancelled",
5337        WorkflowTerminalKind::Terminated => "terminated",
5338        WorkflowTerminalKind::TimedOut => "timed_out",
5339    };
5340    let failure = description
5341        .failure
5342        .as_ref()
5343        .filter(|value| value.is_object());
5344    let nested_failure = failure
5345        .and_then(|value| value.get("failures"))
5346        .and_then(Value::as_array)
5347        .and_then(|failures| failures.last())
5348        .or_else(|| description.failures.last());
5349    let exception = description
5350        .exception
5351        .clone()
5352        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5353        .or_else(|| {
5354            nested_failure
5355                .and_then(|value| value.get("exception_payload"))
5356                .cloned()
5357        });
5358    let string_field = |name: &str| {
5359        failure
5360            .and_then(|value| value.get(name))
5361            .and_then(Value::as_str)
5362            .or_else(|| {
5363                nested_failure
5364                    .and_then(|value| value.get(name))
5365                    .and_then(Value::as_str)
5366            })
5367            .map(str::to_string)
5368    };
5369    let exception_field = |name: &str| {
5370        exception
5371            .as_ref()
5372            .and_then(|value| value.get(name))
5373            .and_then(Value::as_str)
5374            .map(str::to_string)
5375    };
5376    let message = description
5377        .error
5378        .clone()
5379        .or_else(|| string_field("message"))
5380        .or_else(|| exception_field("message"));
5381    let reason = description
5382        .raw
5383        .get("reason")
5384        .and_then(Value::as_str)
5385        .map(str::to_string)
5386        .or_else(|| {
5387            failure
5388                .and_then(|value| value.get("reason"))
5389                .and_then(Value::as_str)
5390                .map(str::to_string)
5391        })
5392        .or_else(|| description.closed_reason.clone())
5393        .unwrap_or_else(|| default_reason.to_string());
5394    let failure_id = string_field("failure_id").or_else(|| {
5395        nested_failure
5396            .and_then(|value| value.get("id"))
5397            .and_then(Value::as_str)
5398            .map(str::to_string)
5399    });
5400
5401    WorkflowTerminalOutcome {
5402        kind,
5403        workflow_id: description
5404            .workflow_id
5405            .clone()
5406            .unwrap_or_else(|| workflow_id.to_string()),
5407        run_id: description
5408            .run_id
5409            .clone()
5410            .or_else(|| run_id.map(str::to_string)),
5411        reason,
5412        failure_category: string_field("failure_category")
5413            .or_else(|| Some(default_reason.to_string())),
5414        failure_id,
5415        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5416        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5417        non_retryable: failure
5418            .and_then(|value| value.get("non_retryable"))
5419            .and_then(Value::as_bool)
5420            .or_else(|| {
5421                nested_failure
5422                    .and_then(|value| value.get("non_retryable"))
5423                    .and_then(Value::as_bool)
5424            }),
5425        message,
5426        exception,
5427        raw: description.raw_value(),
5428    }
5429}
5430
5431#[derive(Clone, Debug, Deserialize)]
5432pub struct RegisterWorkerResponse {
5433    pub worker_id: String,
5434    pub registered: bool,
5435    #[serde(default)]
5436    pub heartbeat_interval_seconds: Option<u64>,
5437    #[serde(default)]
5438    pub protocol_version: Option<String>,
5439    #[serde(default)]
5440    pub server_capabilities: Option<Value>,
5441}
5442
5443/// Result of gracefully removing a worker-plane registration.
5444#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5445pub struct WorkerDeregistrationEnvelope {
5446    pub worker_id: String,
5447    pub outcome: String,
5448    pub recovered_workflow_task_count: u64,
5449}
5450
5451#[derive(Clone, Debug, Deserialize)]
5452pub struct PollWorkflowTaskResponse {
5453    #[serde(default)]
5454    pub task: Option<WorkflowTask>,
5455    #[serde(default)]
5456    pub poll_status: Option<String>,
5457    #[serde(default)]
5458    pub reason: Option<String>,
5459    #[serde(default)]
5460    pub protocol_version: Option<String>,
5461    #[serde(default)]
5462    pub server_capabilities: Option<Value>,
5463}
5464
5465impl PollWorkflowTaskResponse {
5466    /// Classify this response without parsing server display text.
5467    pub fn outcome(&self) -> WorkerPollOutcome {
5468        worker_poll_outcome(
5469            self.task.is_some(),
5470            self.poll_status.as_deref(),
5471            self.reason.as_deref(),
5472        )
5473    }
5474}
5475
5476fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5477    let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5478        return false;
5479    };
5480    let supported = capabilities
5481        .get("workflow_memo_updates")
5482        .and_then(Value::as_object)
5483        .and_then(|memo| memo.get("supported"))
5484        .and_then(Value::as_bool)
5485        == Some(true);
5486    let command_advertised = capabilities
5487        .get("supported_workflow_task_commands")
5488        .and_then(Value::as_array)
5489        .is_some_and(|commands| {
5490            commands
5491                .iter()
5492                .any(|command| command.as_str() == Some("upsert_memo"))
5493        });
5494    supported && command_advertised
5495}
5496
5497fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5498    commands
5499        .iter()
5500        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5501}
5502
5503#[derive(Clone, Debug, Deserialize)]
5504pub struct PollActivityTaskResponse {
5505    #[serde(default)]
5506    pub task: Option<ActivityTask>,
5507    #[serde(default)]
5508    pub poll_status: Option<String>,
5509    #[serde(default)]
5510    pub reason: Option<String>,
5511}
5512
5513impl PollActivityTaskResponse {
5514    /// Classify this response without parsing server display text.
5515    pub fn outcome(&self) -> WorkerPollOutcome {
5516        worker_poll_outcome(
5517            self.task.is_some(),
5518            self.poll_status.as_deref(),
5519            self.reason.as_deref(),
5520        )
5521    }
5522}
5523
5524#[derive(Clone, Debug, Deserialize)]
5525pub struct PollQueryTaskResponse {
5526    #[serde(default)]
5527    pub task: Option<QueryTask>,
5528    #[serde(default)]
5529    pub poll_status: Option<String>,
5530    #[serde(default)]
5531    pub reason: Option<String>,
5532}
5533
5534impl PollQueryTaskResponse {
5535    /// Classify this response without parsing server display text.
5536    pub fn outcome(&self) -> WorkerPollOutcome {
5537        worker_poll_outcome(
5538            self.task.is_some(),
5539            self.poll_status.as_deref(),
5540            self.reason.as_deref(),
5541        )
5542    }
5543}
5544
5545/// Stable classification for worker poll responses.
5546#[derive(Clone, Debug, PartialEq, Eq)]
5547pub enum WorkerPollOutcome {
5548    /// A task was leased and is available on the response.
5549    Task,
5550    /// No task was leased, but the worker should continue polling.
5551    Idle {
5552        poll_status: Option<String>,
5553        reason: Option<String>,
5554    },
5555    /// The server asked this worker to stop claiming new work.
5556    Stop {
5557        poll_status: Option<String>,
5558        reason: Option<String>,
5559    },
5560}
5561
5562impl WorkerPollOutcome {
5563    pub fn should_stop(&self) -> bool {
5564        matches!(self, Self::Stop { .. })
5565    }
5566}
5567
5568fn worker_poll_outcome(
5569    has_task: bool,
5570    poll_status: Option<&str>,
5571    reason: Option<&str>,
5572) -> WorkerPollOutcome {
5573    if worker_poll_is_stop(poll_status, reason) {
5574        return WorkerPollOutcome::Stop {
5575            poll_status: poll_status.map(str::to_string),
5576            reason: reason.map(str::to_string),
5577        };
5578    }
5579
5580    if has_task {
5581        WorkerPollOutcome::Task
5582    } else {
5583        WorkerPollOutcome::Idle {
5584            poll_status: poll_status.map(str::to_string),
5585            reason: reason.map(str::to_string),
5586        }
5587    }
5588}
5589
5590/// An ephemeral server-routed query task.
5591#[derive(Clone, Debug, Deserialize)]
5592pub struct QueryTask {
5593    pub query_task_id: String,
5594    #[serde(default = "default_workflow_task_attempt")]
5595    pub query_task_attempt: u64,
5596    #[serde(default)]
5597    pub lease_owner: Option<String>,
5598    #[serde(default)]
5599    pub workflow_id: Option<String>,
5600    #[serde(default)]
5601    pub run_id: Option<String>,
5602    pub workflow_type: String,
5603    pub query_name: String,
5604    #[serde(
5605        default = "missing_task_payload_codec",
5606        deserialize_with = "deserialize_task_payload_codec"
5607    )]
5608    pub payload_codec: String,
5609    #[serde(default)]
5610    pub workflow_arguments: Option<Value>,
5611    #[serde(default)]
5612    pub query_arguments: Option<Value>,
5613    #[serde(default)]
5614    pub history_events: Vec<HistoryEvent>,
5615    #[serde(default)]
5616    pub history_export: Option<Value>,
5617    #[serde(default)]
5618    pub run_status: Option<String>,
5619}
5620
5621#[derive(Clone, Debug, Deserialize)]
5622pub struct WorkflowTask {
5623    pub task_id: String,
5624    #[serde(default)]
5625    pub workflow_command_id: Option<String>,
5626    #[serde(default)]
5627    pub workflow_id: Option<String>,
5628    #[serde(default)]
5629    pub run_id: Option<String>,
5630    pub workflow_type: String,
5631    #[serde(default)]
5632    pub cancel_requested: bool,
5633    #[serde(
5634        default = "missing_task_payload_codec",
5635        deserialize_with = "deserialize_task_payload_codec"
5636    )]
5637    pub payload_codec: String,
5638    #[serde(default)]
5639    pub arguments: Option<Value>,
5640    #[serde(default)]
5641    pub history_events: Vec<HistoryEvent>,
5642    #[serde(default)]
5643    pub total_history_events: Option<u64>,
5644    #[serde(default)]
5645    pub history_size_bytes: Option<u64>,
5646    #[serde(default)]
5647    pub continue_as_new_recommended: Option<bool>,
5648    #[serde(default)]
5649    pub history_budget_pressure: Option<String>,
5650    #[serde(default)]
5651    pub next_history_page_token: Option<String>,
5652    #[serde(default = "default_workflow_task_attempt")]
5653    pub workflow_task_attempt: u64,
5654    #[serde(default)]
5655    pub workflow_signal_id: Option<String>,
5656    #[serde(default)]
5657    pub signal_name: Option<String>,
5658    #[serde(default)]
5659    pub signal_arguments: Option<Value>,
5660    #[serde(default)]
5661    pub workflow_update_id: Option<String>,
5662    #[serde(default)]
5663    pub update_name: Option<String>,
5664    #[serde(default)]
5665    pub lease_owner: Option<String>,
5666}
5667
5668impl WorkflowTask {
5669    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5670        self.history_events.extend(page.history_events);
5671
5672        if page.total_history_events.is_some() {
5673            self.total_history_events = page.total_history_events;
5674        }
5675
5676        self.next_history_page_token = page
5677            .next_history_page_token
5678            .filter(|token| !token.is_empty());
5679    }
5680}
5681
5682#[derive(Clone, Debug, Deserialize)]
5683struct WorkflowTaskHistoryPage {
5684    #[serde(default)]
5685    history_events: Vec<HistoryEvent>,
5686    #[serde(default)]
5687    total_history_events: Option<u64>,
5688    #[serde(default)]
5689    next_history_page_token: Option<String>,
5690}
5691
5692#[derive(Clone, Debug, Deserialize)]
5693pub struct ActivityTask {
5694    pub task_id: String,
5695    #[serde(default)]
5696    pub activity_attempt_id: Option<String>,
5697    #[serde(default)]
5698    pub attempt_id: Option<String>,
5699    pub activity_type: String,
5700    #[serde(
5701        default = "missing_task_payload_codec",
5702        deserialize_with = "deserialize_task_payload_codec"
5703    )]
5704    pub payload_codec: String,
5705    #[serde(default)]
5706    pub arguments: Option<Value>,
5707    #[serde(default = "default_attempt_number")]
5708    pub attempt_number: u64,
5709    #[serde(default)]
5710    pub lease_owner: Option<String>,
5711}
5712
5713#[derive(Clone, Debug, Deserialize)]
5714pub struct HistoryEvent {
5715    #[serde(alias = "type")]
5716    pub event_type: String,
5717    #[serde(default)]
5718    pub payload: Value,
5719    #[serde(flatten)]
5720    pub raw: HashMap<String, Value>,
5721}
5722
5723/// One decoded signal in the committed workflow-history snapshot.
5724#[derive(Clone, Debug, PartialEq)]
5725pub struct QuerySignal {
5726    pub id: Option<String>,
5727    pub name: String,
5728    pub arguments: Vec<Value>,
5729    avro_arguments: Vec<AvroValue>,
5730    pub workflow_sequence: Option<u64>,
5731}
5732
5733impl QuerySignal {
5734    /// Lossless fixed Avro Value arguments for this committed signal.
5735    pub fn arguments_avro_value(&self) -> &[AvroValue] {
5736        &self.avro_arguments
5737    }
5738}
5739
5740/// Immutable state supplied to a registered query handler.
5741///
5742/// This context intentionally exposes no activity, signal-wait, or command
5743/// APIs. Query handlers inspect committed history and return a value; query
5744/// completion does not append an event or advance deterministic execution.
5745#[derive(Clone, Debug)]
5746pub struct QueryContext {
5747    pub workflow_id: Option<String>,
5748    pub run_id: Option<String>,
5749    pub workflow_type: String,
5750    pub run_status: Option<String>,
5751    workflow_input: Value,
5752    workflow_input_avro_value: AvroValue,
5753    history_events: Arc<Vec<HistoryEvent>>,
5754    signal_events: Arc<Vec<QuerySignal>>,
5755}
5756
5757impl QueryContext {
5758    /// The normalized argument list used to start the workflow.
5759    pub fn workflow_input(&self) -> &Value {
5760        &self.workflow_input
5761    }
5762
5763    /// The lossless fixed Avro Value argument list used to start the workflow.
5764    pub fn workflow_input_avro_value(&self) -> &AvroValue {
5765        &self.workflow_input_avro_value
5766    }
5767
5768    /// The immutable committed history used for this query snapshot.
5769    pub fn history_events(&self) -> &[HistoryEvent] {
5770        self.history_events.as_slice()
5771    }
5772
5773    /// All decoded signals in committed workflow order.
5774    pub fn signal_events(&self) -> &[QuerySignal] {
5775        self.signal_events.as_slice()
5776    }
5777
5778    /// Decoded argument lists for each committed signal with `signal_name`.
5779    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5780        self.signal_events
5781            .iter()
5782            .filter(|signal| signal.name == signal_name)
5783            .map(|signal| signal.arguments.clone())
5784            .collect()
5785    }
5786
5787    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
5788    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5789        self.signal_events
5790            .iter()
5791            .filter(|signal| signal.name == signal_name)
5792            .map(|signal| signal.avro_arguments.clone())
5793            .collect()
5794    }
5795}
5796
5797#[derive(Clone, Debug, Deserialize)]
5798pub struct ActivityHeartbeatResponse {
5799    #[serde(default)]
5800    pub cancel_requested: bool,
5801    #[serde(default)]
5802    pub heartbeat_recorded: bool,
5803    #[serde(default)]
5804    pub can_continue: Option<bool>,
5805    #[serde(default)]
5806    pub reason: Option<String>,
5807    #[serde(default)]
5808    pub run_closed_reason: Option<String>,
5809    #[serde(default)]
5810    pub run_closed_at: Option<String>,
5811    #[serde(default)]
5812    pub lease_expires_at: Option<String>,
5813    #[serde(default)]
5814    pub last_heartbeat_at: Option<String>,
5815}
5816
5817impl ActivityHeartbeatResponse {
5818    /// Whether the activity should stop instead of attempting completion.
5819    pub fn should_stop(&self) -> bool {
5820        self.cancel_requested || self.can_continue == Some(false)
5821    }
5822}
5823
5824fn missing_task_payload_codec() -> String {
5825    MISSING_TASK_PAYLOAD_CODEC.to_string()
5826}
5827
5828fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5829where
5830    D: Deserializer<'de>,
5831{
5832    Ok(match Value::deserialize(deserializer)? {
5833        Value::String(codec) => codec,
5834        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5835        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5836    })
5837}
5838
5839fn default_workflow_task_attempt() -> u64 {
5840    1
5841}
5842
5843fn default_attempt_number() -> u64 {
5844    1
5845}
5846
5847type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5848type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5849type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5850type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5851type ReplayedWorkflowHandler =
5852    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5853type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5854type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5855type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5856type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5857type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5858type ReplayedQueryHandler = Arc<
5859    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5860        + Send
5861        + Sync,
5862>;
5863type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5864
5865struct ReplayedWorkflowInvocation {
5866    future: WorkflowFuture,
5867    snapshot: WorkflowStateSnapshot,
5868}
5869
5870#[derive(Clone)]
5871struct RegisteredWorkflow {
5872    execute: WorkflowHandler,
5873    replay: Option<ReplayedWorkflowHandler>,
5874    state_type: Option<TypeId>,
5875}
5876
5877#[derive(Debug)]
5878struct WorkflowTaskDecision {
5879    commands: Vec<Value>,
5880    message_stream_cursors: Vec<Value>,
5881    message_stream_waits: Vec<Value>,
5882}
5883
5884impl WorkflowTaskDecision {
5885    fn without_message_streams(commands: Vec<Value>) -> Self {
5886        Self {
5887            commands,
5888            message_stream_cursors: Vec::new(),
5889            message_stream_waits: Vec::new(),
5890        }
5891    }
5892}
5893
5894#[derive(Clone)]
5895enum RegisteredQuery {
5896    Snapshot(QueryHandler),
5897    Replayed {
5898        state_type: TypeId,
5899        handler: ReplayedQueryHandler,
5900    },
5901}
5902
5903#[derive(Clone, Debug)]
5904pub struct WorkerHeartbeatObservation {
5905    pub worker_id: String,
5906    pub task_queue: String,
5907    pub acknowledged_at_unix_millis: u64,
5908    pub acknowledgement: Value,
5909}
5910
5911/// Bounded retry policy for worker poll acquisition and worker heartbeats.
5912///
5913/// Expected empty long polls and explicit Server capacity backpressure are
5914/// normal worker states. Capacity backpressure honors the advertised retry
5915/// delay without consuming this retry budget. Other transport failures,
5916/// retryable HTTP 408/429 responses, and server errors are retried with capped
5917/// exponential backoff. Authentication, protocol, codec, and handler failures
5918/// are never retried by the worker.
5919/// Explicit storage admission pauses also preserve the already-serialized worker
5920/// request without consuming this budget or rerunning its handler. The delay is
5921/// capped by `max_backoff`, and shutdown interrupts storage waits.
5922#[derive(Clone, Copy, Debug)]
5923pub struct WorkerRetryPolicy {
5924    /// Number of retries after the initial request fails.
5925    pub max_retries: usize,
5926    /// Delay before the first retry.
5927    pub initial_backoff: Duration,
5928    /// Maximum delay between retries.
5929    pub max_backoff: Duration,
5930}
5931
5932impl Default for WorkerRetryPolicy {
5933    fn default() -> Self {
5934        Self {
5935            max_retries: 5,
5936            initial_backoff: Duration::from_millis(100),
5937            max_backoff: Duration::from_secs(5),
5938        }
5939    }
5940}
5941
5942#[derive(Clone, Debug)]
5943struct WorkerStorageAdmission {
5944    policy: WorkerRetryPolicy,
5945    stop: Arc<AtomicBool>,
5946}
5947
5948struct StopWorkerOnDrop(Arc<AtomicBool>);
5949
5950impl Drop for StopWorkerOnDrop {
5951    fn drop(&mut self) {
5952        self.0.store(true, Ordering::SeqCst);
5953    }
5954}
5955
5956async fn wait_for_worker_stop(stop: &AtomicBool) {
5957    while !stop.load(Ordering::SeqCst) {
5958        tokio::time::sleep(Duration::from_millis(100)).await;
5959    }
5960}
5961
5962#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5963enum ManagedPollOutcome {
5964    Idle,
5965    Handled,
5966    Stop,
5967}
5968
5969#[derive(Clone)]
5970pub struct Worker {
5971    client: Client,
5972    worker_id: String,
5973    task_queue: String,
5974    workflows: HashMap<String, RegisteredWorkflow>,
5975    activities: HashMap<String, ActivityHandler>,
5976    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5977    updates: HashMap<String, HashMap<String, UpdateHandler>>,
5978    max_concurrent_workflow_tasks: usize,
5979    max_concurrent_activity_tasks: usize,
5980    poll_timeout: Duration,
5981    heartbeat_interval: Duration,
5982    retry_policy: WorkerRetryPolicy,
5983    heartbeat_observer: Option<WorkerHeartbeatObserver>,
5984}
5985
5986impl Worker {
5987    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5988        Self {
5989            client,
5990            worker_id: default_worker_id(),
5991            task_queue: task_queue.into(),
5992            workflows: HashMap::new(),
5993            activities: HashMap::new(),
5994            queries: HashMap::new(),
5995            updates: HashMap::new(),
5996            max_concurrent_workflow_tasks: 10,
5997            max_concurrent_activity_tasks: 10,
5998            poll_timeout: Duration::from_secs(30),
5999            heartbeat_interval: Duration::from_secs(60),
6000            retry_policy: WorkerRetryPolicy::default(),
6001            heartbeat_observer: None,
6002        }
6003    }
6004
6005    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
6006        self.worker_id = worker_id.into();
6007        self
6008    }
6009
6010    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
6011        self.poll_timeout = timeout;
6012        self
6013    }
6014
6015    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
6016        self.heartbeat_interval = interval;
6017        self
6018    }
6019
6020    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
6021    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
6022        self.retry_policy = policy;
6023        self
6024    }
6025
6026    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
6027    where
6028        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
6029    {
6030        self.heartbeat_observer = Some(Arc::new(observer));
6031        self
6032    }
6033
6034    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
6035        self.max_concurrent_workflow_tasks = count.max(1);
6036        self
6037    }
6038
6039    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
6040        self.max_concurrent_activity_tasks = count.max(1);
6041        self
6042    }
6043
6044    /// Register a workflow handler.
6045    ///
6046    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
6047    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
6048    /// while acquiring or decoding a worker task remain worker-operation
6049    /// failures and do not get converted into workflow outcomes.
6050    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
6051    where
6052        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
6053        Fut: Future<Output = Result<Value>> + Send + 'static,
6054    {
6055        let handler = Arc::new(handler);
6056        self.workflows.insert(
6057            workflow_type.into(),
6058            RegisteredWorkflow {
6059                execute: Arc::new(move |ctx, input| {
6060                    let handler = Arc::clone(&handler);
6061                    Box::pin(async move {
6062                        let result = handler(ctx, input.into_json()?).await?;
6063                        AvroValue::from_serialize(&result)
6064                    })
6065                }),
6066                replay: None,
6067                state_type: None,
6068            },
6069        );
6070    }
6071
6072    /// Register a workflow with one Serde request value and a Serde result.
6073    ///
6074    /// This is an ergonomic adapter over the same fixed Avro Value protocol as
6075    /// [`Worker::register_workflow_avro_value`]. It does not create or publish a
6076    /// workflow-specific schema. A task must contain zero arguments for a unit
6077    /// request or exactly one argument for every other request type.
6078    ///
6079    /// See the runnable
6080    /// [`hello_world` example](https://github.com/durable-workflow/sdk-rust/blob/main/examples/hello_world.rs)
6081    /// for typed workflow and activity contracts with retry and timeout policy.
6082    pub fn register_typed_workflow<I, O, F, Fut>(
6083        &mut self,
6084        workflow_type: impl Into<String>,
6085        handler: F,
6086    ) where
6087        I: DeserializeOwned + Send + 'static,
6088        O: Serialize + Send + 'static,
6089        F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6090        Fut: Future<Output = Result<O>> + Send + 'static,
6091    {
6092        let workflow_type = workflow_type.into();
6093        let handler_name = workflow_type.clone();
6094        let handler = Arc::new(handler);
6095        self.workflows.insert(
6096            workflow_type,
6097            RegisteredWorkflow {
6098                execute: Arc::new(move |ctx, input| {
6099                    let handler = Arc::clone(&handler);
6100                    let handler_name = handler_name.clone();
6101                    Box::pin(async move {
6102                        let input =
6103                            decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6104                        let result = handler(ctx, input).await?;
6105                        encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6106                    })
6107                }),
6108                replay: None,
6109                state_type: None,
6110            },
6111        );
6112    }
6113
6114    /// Register a workflow on the lossless fixed Avro Value surface.
6115    pub fn register_workflow_avro_value<F, Fut>(
6116        &mut self,
6117        workflow_type: impl Into<String>,
6118        handler: F,
6119    ) where
6120        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6121        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6122    {
6123        self.workflows.insert(
6124            workflow_type.into(),
6125            RegisteredWorkflow {
6126                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6127                replay: None,
6128                state_type: None,
6129            },
6130        );
6131    }
6132
6133    /// Register a workflow whose typed instance state can be reconstructed for queries.
6134    ///
6135    /// `state_factory` creates a fresh instance for every normal workflow task and
6136    /// query replay. The workflow handler is the single source of truth for state
6137    /// transitions: it updates [`WorkflowInstance`] after activities and signals
6138    /// resolve. Query replay runs this same handler over committed history and
6139    /// discards any commands it would emit.
6140    pub fn register_replayed_workflow<S, Factory, F, Fut>(
6141        &mut self,
6142        workflow_type: impl Into<String>,
6143        state_factory: Factory,
6144        handler: F,
6145    ) where
6146        S: Clone + Send + Sync + 'static,
6147        Factory: Fn() -> S + Send + Sync + 'static,
6148        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6149        Fut: Future<Output = Result<Value>> + Send + 'static,
6150    {
6151        let state_factory = Arc::new(state_factory);
6152        let handler = Arc::new(handler);
6153
6154        let execute_factory = Arc::clone(&state_factory);
6155        let execute_handler = Arc::clone(&handler);
6156        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6157            let state = WorkflowInstance::new(execute_factory());
6158            let handler = Arc::clone(&execute_handler);
6159            Box::pin(async move {
6160                let result = handler(ctx, input.into_json()?, state).await?;
6161                AvroValue::from_serialize(&result)
6162            }) as WorkflowFuture
6163        });
6164
6165        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6166            let state = WorkflowInstance::new(state_factory());
6167            let snapshot_state = state.clone();
6168            let snapshot: WorkflowStateSnapshot =
6169                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6170            let replay_handler = Arc::clone(&handler);
6171            let future = async move {
6172                let result = replay_handler(ctx, input.into_json()?, state).await?;
6173                AvroValue::from_serialize(&result)
6174            };
6175            ReplayedWorkflowInvocation {
6176                future: Box::pin(future),
6177                snapshot,
6178            }
6179        });
6180
6181        self.workflows.insert(
6182            workflow_type.into(),
6183            RegisteredWorkflow {
6184                execute,
6185                replay: Some(replay),
6186                state_type: Some(TypeId::of::<S>()),
6187            },
6188        );
6189    }
6190
6191    /// Register a replayable workflow with one Serde request value and result.
6192    ///
6193    /// Normal task execution and instance-state query replay both decode and
6194    /// encode through the fixed Avro Value codec. The state factory and handler
6195    /// otherwise follow [`Worker::register_replayed_workflow`].
6196    pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6197        &mut self,
6198        workflow_type: impl Into<String>,
6199        state_factory: Factory,
6200        handler: F,
6201    ) where
6202        I: DeserializeOwned + Send + 'static,
6203        O: Serialize + Send + 'static,
6204        S: Clone + Send + Sync + 'static,
6205        Factory: Fn() -> S + Send + Sync + 'static,
6206        F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6207        Fut: Future<Output = Result<O>> + Send + 'static,
6208    {
6209        let workflow_type = workflow_type.into();
6210        let state_factory = Arc::new(state_factory);
6211        let handler = Arc::new(handler);
6212
6213        let execute_name = workflow_type.clone();
6214        let execute_factory = Arc::clone(&state_factory);
6215        let execute_handler = Arc::clone(&handler);
6216        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6217            let state = WorkflowInstance::new(execute_factory());
6218            let handler = Arc::clone(&execute_handler);
6219            let handler_name = execute_name.clone();
6220            Box::pin(async move {
6221                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6222                let result = handler(ctx, input, state).await?;
6223                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6224            }) as WorkflowFuture
6225        });
6226
6227        let replay_name = workflow_type.clone();
6228        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6229            let state = WorkflowInstance::new(state_factory());
6230            let snapshot_state = state.clone();
6231            let snapshot: WorkflowStateSnapshot =
6232                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6233            let handler = Arc::clone(&handler);
6234            let handler_name = replay_name.clone();
6235            let future = async move {
6236                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6237                let result = handler(ctx, input, state).await?;
6238                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6239            };
6240            ReplayedWorkflowInvocation {
6241                future: Box::pin(future),
6242                snapshot,
6243            }
6244        });
6245
6246        self.workflows.insert(
6247            workflow_type,
6248            RegisteredWorkflow {
6249                execute,
6250                replay: Some(replay),
6251                state_type: Some(TypeId::of::<S>()),
6252            },
6253        );
6254    }
6255
6256    /// Register a replayable workflow on the lossless fixed Avro Value surface.
6257    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6258        &mut self,
6259        workflow_type: impl Into<String>,
6260        state_factory: Factory,
6261        handler: F,
6262    ) where
6263        S: Clone + Send + Sync + 'static,
6264        Factory: Fn() -> S + Send + Sync + 'static,
6265        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6266        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6267    {
6268        let state_factory = Arc::new(state_factory);
6269        let handler = Arc::new(handler);
6270
6271        let execute_factory = Arc::clone(&state_factory);
6272        let execute_handler = Arc::clone(&handler);
6273        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6274            let state = WorkflowInstance::new(execute_factory());
6275            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6276        });
6277
6278        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6279            let state = WorkflowInstance::new(state_factory());
6280            let snapshot_state = state.clone();
6281            let snapshot: WorkflowStateSnapshot =
6282                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6283            ReplayedWorkflowInvocation {
6284                future: Box::pin(handler(ctx, input, state)),
6285                snapshot,
6286            }
6287        });
6288
6289        self.workflows.insert(
6290            workflow_type.into(),
6291            RegisteredWorkflow {
6292                execute,
6293                replay: Some(replay),
6294                state_type: Some(TypeId::of::<S>()),
6295            },
6296        );
6297    }
6298
6299    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6300    where
6301        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6302        Fut: Future<Output = Result<Value>> + Send + 'static,
6303    {
6304        let handler = Arc::new(handler);
6305        self.activities.insert(
6306            activity_type.into(),
6307            Arc::new(move |ctx, args| {
6308                let handler = Arc::clone(&handler);
6309                Box::pin(async move {
6310                    let result = handler(ctx, args.into_json()?).await?;
6311                    AvroValue::from_serialize(&result)
6312                })
6313            }),
6314        );
6315    }
6316
6317    /// Register an activity with one Serde request value and a Serde result.
6318    ///
6319    /// Inputs and results use the platform's fixed Avro Value schema. Shape
6320    /// mismatches and unsupported Serde values return [`Error::HandlerType`]
6321    /// with the activity name and Rust type.
6322    pub fn register_typed_activity<I, O, F, Fut>(
6323        &mut self,
6324        activity_type: impl Into<String>,
6325        handler: F,
6326    ) where
6327        I: DeserializeOwned + Send + 'static,
6328        O: Serialize + Send + 'static,
6329        F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6330        Fut: Future<Output = Result<O>> + Send + 'static,
6331    {
6332        let activity_type = activity_type.into();
6333        let handler_name = activity_type.clone();
6334        let handler = Arc::new(handler);
6335        self.activities.insert(
6336            activity_type,
6337            Arc::new(move |ctx, input| {
6338                let handler = Arc::clone(&handler);
6339                let handler_name = handler_name.clone();
6340                Box::pin(async move {
6341                    let input =
6342                        decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6343                    let result = handler(ctx, input).await?;
6344                    encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6345                })
6346            }),
6347        );
6348    }
6349
6350    /// Register an activity on the lossless fixed Avro Value surface.
6351    pub fn register_activity_avro_value<F, Fut>(
6352        &mut self,
6353        activity_type: impl Into<String>,
6354        handler: F,
6355    ) where
6356        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6357        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6358    {
6359        self.activities.insert(
6360            activity_type.into(),
6361            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6362        );
6363    }
6364
6365    /// Register a named, read-only query handler for a workflow type.
6366    ///
6367    /// The workflow type must also be registered with [`Worker::register_workflow`]
6368    /// before the worker runs. The handler receives only an immutable committed
6369    /// state snapshot and normalized query arguments.
6370    pub fn register_query<F, Fut>(
6371        &mut self,
6372        workflow_type: impl Into<String>,
6373        query_name: impl Into<String>,
6374        handler: F,
6375    ) where
6376        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6377        Fut: Future<Output = Result<Value>> + Send + 'static,
6378    {
6379        let handler = Arc::new(handler);
6380        self.queries
6381            .entry(workflow_type.into())
6382            .or_default()
6383            .insert(
6384                query_name.into(),
6385                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6386                    let handler = Arc::clone(&handler);
6387                    Box::pin(async move {
6388                        let result = handler(ctx, args.into_json()?).await?;
6389                        AvroValue::from_serialize(&result)
6390                    })
6391                })),
6392            );
6393    }
6394
6395    /// Register a query handler on the lossless fixed Avro Value surface.
6396    pub fn register_query_avro_value<F, Fut>(
6397        &mut self,
6398        workflow_type: impl Into<String>,
6399        query_name: impl Into<String>,
6400        handler: F,
6401    ) where
6402        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6403        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6404    {
6405        self.queries
6406            .entry(workflow_type.into())
6407            .or_default()
6408            .insert(
6409                query_name.into(),
6410                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6411            );
6412    }
6413
6414    /// Register a named query against deterministically replayed instance state.
6415    ///
6416    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
6417    /// same state type `S`. The handler receives an immutable, detached state
6418    /// clone, so successful and failed queries cannot affect workflow execution
6419    /// or the state reconstructed by a later query.
6420    pub fn register_replayed_query<S, F, Fut>(
6421        &mut self,
6422        workflow_type: impl Into<String>,
6423        query_name: impl Into<String>,
6424        handler: F,
6425    ) where
6426        S: Clone + Send + Sync + 'static,
6427        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6428        Fut: Future<Output = Result<Value>> + Send + 'static,
6429    {
6430        let handler = Arc::new(handler);
6431        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6432            let state = state.downcast::<S>().map_err(|_| {
6433                "registered query state type does not match the replayed workflow state".to_string()
6434            })?;
6435            let handler = Arc::clone(&handler);
6436            Ok(Box::pin(async move {
6437                let result = handler(ctx, state, args.into_json()?).await?;
6438                AvroValue::from_serialize(&result)
6439            }))
6440        });
6441
6442        self.queries
6443            .entry(workflow_type.into())
6444            .or_default()
6445            .insert(
6446                query_name.into(),
6447                RegisteredQuery::Replayed {
6448                    state_type: TypeId::of::<S>(),
6449                    handler: erased_handler,
6450                },
6451            );
6452    }
6453
6454    /// Register a replayed-state query on the lossless fixed Avro Value surface.
6455    pub fn register_replayed_query_avro_value<S, F, Fut>(
6456        &mut self,
6457        workflow_type: impl Into<String>,
6458        query_name: impl Into<String>,
6459        handler: F,
6460    ) where
6461        S: Clone + Send + Sync + 'static,
6462        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6463        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6464    {
6465        let handler = Arc::new(handler);
6466        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6467            let state = state.downcast::<S>().map_err(|_| {
6468                "registered query state type does not match the replayed workflow state".to_string()
6469            })?;
6470            Ok(Box::pin(handler(ctx, state, args)))
6471        });
6472
6473        self.queries
6474            .entry(workflow_type.into())
6475            .or_default()
6476            .insert(
6477                query_name.into(),
6478                RegisteredQuery::Replayed {
6479                    state_type: TypeId::of::<S>(),
6480                    handler: erased_handler,
6481                },
6482            );
6483    }
6484
6485    /// Register a synchronous workflow update handler.
6486    pub fn register_update<F, Fut>(
6487        &mut self,
6488        workflow_type: impl Into<String>,
6489        update_name: impl Into<String>,
6490        handler: F,
6491    ) where
6492        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6493        Fut: Future<Output = Result<Value>> + Send + 'static,
6494    {
6495        let handler = Arc::new(handler);
6496        self.updates
6497            .entry(workflow_type.into())
6498            .or_default()
6499            .insert(
6500                update_name.into(),
6501                Arc::new(move |ctx, args| {
6502                    let handler = Arc::clone(&handler);
6503                    Box::pin(async move {
6504                        let result = handler(ctx, args.into_json()?).await?;
6505                        AvroValue::from_serialize(&result)
6506                    })
6507                }),
6508            );
6509    }
6510
6511    /// Register an update handler on the lossless fixed Avro Value surface.
6512    pub fn register_update_avro_value<F, Fut>(
6513        &mut self,
6514        workflow_type: impl Into<String>,
6515        update_name: impl Into<String>,
6516        handler: F,
6517    ) where
6518        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6519        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6520    {
6521        self.updates
6522            .entry(workflow_type.into())
6523            .or_default()
6524            .insert(
6525                update_name.into(),
6526                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6527            );
6528    }
6529
6530    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6531        let mut command_contracts = serde_json::Map::new();
6532        for workflow_type in self.workflows.keys() {
6533            let mut queries = self
6534                .queries
6535                .get(workflow_type)
6536                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6537                .unwrap_or_default();
6538            queries.sort();
6539            let mut updates = self
6540                .updates
6541                .get(workflow_type)
6542                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6543                .unwrap_or_default();
6544            updates.sort();
6545            command_contracts.insert(
6546                workflow_type.clone(),
6547                json!({
6548                    "queries": queries,
6549                    "query_contracts": [],
6550                    "signals": [],
6551                    "signal_contracts": [],
6552                    "updates": updates,
6553                    "update_contracts": [],
6554                    "update_validators": [],
6555                }),
6556            );
6557        }
6558
6559        self.client
6560            .register_worker_with_command_contracts(
6561                &self.worker_id,
6562                &self.task_queue,
6563                self.workflows.keys().cloned().collect(),
6564                self.activities.keys().cloned().collect(),
6565                self.max_concurrent_workflow_tasks,
6566                self.max_concurrent_activity_tasks,
6567                [
6568                    Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6569                    Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6570                    Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6571                    Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6572                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6573                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6574                    worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6575                        .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6576                ]
6577                .into_iter()
6578                .flatten()
6579                .collect(),
6580                Value::Object(command_contracts),
6581            )
6582            .await
6583    }
6584
6585    /// Run until shutdown or a terminal worker error occurs.
6586    ///
6587    /// Empty long-poll expirations do not stop the worker. Retryable poll and
6588    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
6589    /// authentication, protocol, and other non-retryable failures are returned.
6590    pub async fn run(&self) -> Result<()> {
6591        self.run_until(std::future::pending::<()>()).await
6592    }
6593
6594    /// Run until `shutdown` resolves or a terminal worker error occurs.
6595    ///
6596    /// This has the same liveness and terminal-error contract as [`Worker::run`].
6597    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6598    where
6599        F: Future<Output = ()>,
6600    {
6601        let stop = Arc::new(AtomicBool::new(false));
6602        let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6603        let worker = self.with_storage_admission(Arc::clone(&stop));
6604        let run = worker.run_with_storage_admission(Arc::clone(&stop));
6605        tokio::pin!(run);
6606        tokio::pin!(shutdown);
6607        tokio::select! {
6608            result = &mut run => result,
6609            _ = &mut shutdown => {
6610                stop.store(true, Ordering::SeqCst);
6611                run.await
6612            }
6613        }
6614    }
6615
6616    fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6617        let mut worker = self.clone();
6618        worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6619            policy: self.retry_policy,
6620            stop,
6621        });
6622        worker
6623    }
6624
6625    async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6626        let registration = self.register().await?;
6627        if !registration.registered {
6628            return Err(Error::WorkerLoop(format!(
6629                "worker registration for {:?} was not accepted",
6630                self.worker_id
6631            )));
6632        }
6633        let registered_worker_id = registration.worker_id.clone();
6634        let primary = self.run_registered_until(stop, registration).await;
6635        let deregistration = self
6636            .client
6637            .deregister_worker_registration(&registered_worker_id)
6638            .await;
6639
6640        match (primary, deregistration) {
6641            (Ok(()), Ok(_)) => Ok(()),
6642            (Ok(()), Err(deregistration)) => Err(deregistration),
6643            (Err(primary), Ok(_)) => Err(primary),
6644            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6645                primary: Box::new(primary),
6646                deregistration: Box::new(deregistration),
6647            }),
6648        }
6649    }
6650
6651    async fn run_registered_until(
6652        &self,
6653        stop: Arc<AtomicBool>,
6654        registration: RegisterWorkerResponse,
6655    ) -> Result<()> {
6656        let heartbeat_interval = Duration::from_secs(
6657            registration
6658                .heartbeat_interval_seconds
6659                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6660        );
6661        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
6662        // from the completion of the preceding attempt, including its bounded
6663        // retries. A fixed-epoch interval can leave an already-due tick queued
6664        // while an acknowledgement is slow, producing a catch-up heartbeat as
6665        // soon as that request completes.
6666        let heartbeat = tokio::time::sleep(Duration::ZERO);
6667        tokio::pin!(heartbeat);
6668        // Poll responses may already have leased server-side work by the time
6669        // they become ready, so each poller owns its responses through
6670        // completion or failure instead of racing raw polls in this select.
6671        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6672            let worker = self.clone();
6673            let stop = Arc::clone(&stop);
6674            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6675        });
6676        let mut activity_poller = (!self.activities.is_empty()).then(|| {
6677            let worker = self.clone();
6678            let stop = Arc::clone(&stop);
6679            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6680        });
6681        let mut query_poller = (!self.queries.is_empty()).then(|| {
6682            let worker = self.clone();
6683            let stop = Arc::clone(&stop);
6684            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6685        });
6686
6687        loop {
6688            tokio::select! {
6689                _ = wait_for_worker_stop(&stop) => {
6690                    stop.store(true, Ordering::SeqCst);
6691                    break;
6692                }
6693                _ = &mut heartbeat => {
6694                    let result = self.retry_worker_operation(|| {
6695                        self.client.heartbeat_worker(
6696                            &self.worker_id,
6697                            self.max_concurrent_workflow_tasks,
6698                            self.max_concurrent_activity_tasks,
6699                        )
6700                    }).await;
6701                    heartbeat
6702                        .as_mut()
6703                        .reset(tokio::time::Instant::now() + heartbeat_interval);
6704                    match result {
6705                        Ok(acknowledgement) => {
6706                            if let Some(observer) = &self.heartbeat_observer {
6707                                observer(&WorkerHeartbeatObservation {
6708                                    worker_id: self.worker_id.clone(),
6709                                    task_queue: self.task_queue.clone(),
6710                                    acknowledged_at_unix_millis: SystemTime::now()
6711                                        .duration_since(UNIX_EPOCH)
6712                                        .unwrap_or_default()
6713                                        .as_millis()
6714                                        .min(u64::MAX as u128)
6715                                        as u64,
6716                                    acknowledgement,
6717                                });
6718                            }
6719                        }
6720                        Err(error) => {
6721                            stop.store(true, Ordering::SeqCst);
6722                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6723                            return Err(error);
6724                        }
6725                    }
6726                }
6727                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6728                    workflow_poller = None;
6729                    let stopped_by_server = stop.load(Ordering::SeqCst);
6730                    stop.store(true, Ordering::SeqCst);
6731                    let poller_result = optional_poller_result("workflow", result);
6732                    let join_result =
6733                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6734                    poller_result?;
6735                    join_result?;
6736                    if stopped_by_server {
6737                        return Ok(());
6738                    }
6739                    return Err(Error::WorkerLoop(
6740                        "workflow poller stopped unexpectedly".to_string(),
6741                    ));
6742                }
6743                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6744                    activity_poller = None;
6745                    let stopped_by_server = stop.load(Ordering::SeqCst);
6746                    stop.store(true, Ordering::SeqCst);
6747                    let poller_result = optional_poller_result("activity", result);
6748                    let join_result =
6749                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6750                    poller_result?;
6751                    join_result?;
6752                    if stopped_by_server {
6753                        return Ok(());
6754                    }
6755                    return Err(Error::WorkerLoop(
6756                        "activity poller stopped unexpectedly".to_string(),
6757                    ));
6758                }
6759                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6760                    query_poller = None;
6761                    let stopped_by_server = stop.load(Ordering::SeqCst);
6762                    stop.store(true, Ordering::SeqCst);
6763                    let poller_result = optional_poller_result("query", result);
6764                    let join_result =
6765                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6766                    poller_result?;
6767                    join_result?;
6768                    if stopped_by_server {
6769                        return Ok(());
6770                    }
6771                    return Err(Error::WorkerLoop(
6772                        "query poller stopped unexpectedly".to_string(),
6773                    ));
6774                }
6775            }
6776        }
6777
6778        join_pollers(
6779            workflow_poller.take(),
6780            activity_poller.take(),
6781            query_poller.take(),
6782        )
6783        .await
6784    }
6785
6786    /// Poll and settle at most one task from each enabled task family.
6787    ///
6788    /// A workflow may reach its server-enforced run deadline while this worker
6789    /// holds a task. When the completion endpoint authoritatively rejects that
6790    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
6791    /// terminal `run_status=failed`, the workflow tick is considered settled:
6792    /// the late command was not recorded and cannot replace the terminal run.
6793    /// Every other completion rejection remains an error. This worker-level
6794    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
6795    /// only bounds how long a client waits for a result.
6796    ///
6797    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
6798    /// the original [`Error::Http`] status and response body.
6799    pub async fn run_once(&self) -> Result<usize> {
6800        let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
6801        let mut handled = 0;
6802        match worker.poll_workflow_once().await? {
6803            ManagedPollOutcome::Handled => handled += 1,
6804            ManagedPollOutcome::Stop => return Ok(handled),
6805            ManagedPollOutcome::Idle => {}
6806        }
6807        match worker.poll_activity_once().await? {
6808            ManagedPollOutcome::Handled => handled += 1,
6809            ManagedPollOutcome::Stop => return Ok(handled),
6810            ManagedPollOutcome::Idle => {}
6811        }
6812        if !self.queries.is_empty() {
6813            match worker.poll_query_once().await? {
6814                ManagedPollOutcome::Handled => handled += 1,
6815                ManagedPollOutcome::Stop => return Ok(handled),
6816                ManagedPollOutcome::Idle => {}
6817            }
6818        }
6819        Ok(handled)
6820    }
6821
6822    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6823        let poll_request_id = unique_request_id("rust-workflow-poll");
6824        let response = self
6825            .retry_worker_operation(|| {
6826                self.client.poll_workflow_task_response_with_request_id(
6827                    &self.worker_id,
6828                    &self.task_queue,
6829                    self.poll_timeout,
6830                    &poll_request_id,
6831                    0,
6832                )
6833            })
6834            .await;
6835        let Some(response) = self.settle_worker_poll_response(response).await? else {
6836            return Ok(ManagedPollOutcome::Idle);
6837        };
6838        if response.outcome().should_stop() {
6839            return Ok(ManagedPollOutcome::Stop);
6840        }
6841        let memo_updates_supported =
6842            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6843        let Some(task) = response.task else {
6844            return Ok(ManagedPollOutcome::Idle);
6845        };
6846
6847        let task_id = task.task_id.clone();
6848        let attempt = task.workflow_task_attempt;
6849        let run_id = task.run_id.clone();
6850        let lease_owner = task
6851            .lease_owner
6852            .clone()
6853            .unwrap_or_else(|| self.worker_id.clone());
6854
6855        match self.execute_workflow_task_decision(task) {
6856            Ok(decision)
6857                if commands_use_workflow_memo_updates(&decision.commands)
6858                    && !memo_updates_supported =>
6859            {
6860                self.client
6861                    .fail_workflow_task(
6862                        &task_id,
6863                        &lease_owner,
6864                        attempt,
6865                        Error::WorkflowMemoUpdatesUnavailable.to_string(),
6866                    )
6867                    .await?;
6868            }
6869            Ok(decision) if decision.commands.is_empty() => {
6870                // A replay can consume a recorded pending durable command
6871                // without producing a new command. The standalone protocol
6872                // acknowledges that state through the typed waiting outcome;
6873                // an empty completion is rejected by servers that require at
6874                // least one executable command.
6875                self.client
6876                    .fail_workflow_task_with_type(
6877                        &task_id,
6878                        &lease_owner,
6879                        attempt,
6880                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6881                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6882                    )
6883                    .await?;
6884            }
6885            Ok(decision) => {
6886                let completion = self
6887                    .client
6888                    .complete_workflow_task_with_message_streams(
6889                        &task_id,
6890                        &lease_owner,
6891                        attempt,
6892                        decision.commands,
6893                        decision.message_stream_cursors,
6894                        decision.message_stream_waits,
6895                    )
6896                    .await;
6897                if let Err(error) = completion {
6898                    if !workflow_task_completion_is_terminal_timeout(
6899                        &error,
6900                        &task_id,
6901                        attempt,
6902                        run_id.as_deref(),
6903                    ) {
6904                        return Err(error);
6905                    }
6906                }
6907            }
6908            Err(error) => {
6909                self.client
6910                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6911                    .await?;
6912            }
6913        }
6914
6915        Ok(ManagedPollOutcome::Handled)
6916    }
6917
6918    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6919        while !stop.load(Ordering::SeqCst) {
6920            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6921                stop.store(true, Ordering::SeqCst);
6922                break;
6923            }
6924        }
6925
6926        Ok(())
6927    }
6928
6929    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6930        let poll_request_id = unique_request_id("rust-activity-poll");
6931        let response = self
6932            .retry_worker_operation(|| {
6933                self.client.poll_activity_task_response_with_request_id(
6934                    &self.worker_id,
6935                    &self.task_queue,
6936                    self.poll_timeout,
6937                    &poll_request_id,
6938                    0,
6939                )
6940            })
6941            .await;
6942        let Some(response) = self.settle_worker_poll_response(response).await? else {
6943            return Ok(ManagedPollOutcome::Idle);
6944        };
6945        if response.outcome().should_stop() {
6946            return Ok(ManagedPollOutcome::Stop);
6947        }
6948        let Some(task) = response.task else {
6949            return Ok(ManagedPollOutcome::Idle);
6950        };
6951
6952        let task_id = task.task_id.clone();
6953        let attempt_id = task
6954            .activity_attempt_id
6955            .clone()
6956            .or(task.attempt_id.clone())
6957            .unwrap_or_default();
6958        let lease_owner = task
6959            .lease_owner
6960            .clone()
6961            .unwrap_or_else(|| self.worker_id.clone());
6962        let codec = task.payload_codec.clone();
6963        let result = self.execute_activity_task(task).await;
6964        match result {
6965            Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
6966            Ok(value) => {
6967                let completion = self
6968                    .client
6969                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6970                    .await;
6971                if let Err(error) = completion {
6972                    if !activity_task_rejection_is_final(&error) {
6973                        return Err(error);
6974                    }
6975                }
6976            }
6977            Err(error) => {
6978                let failure = self
6979                    .client
6980                    .fail_activity_task(
6981                        &task_id,
6982                        &attempt_id,
6983                        &lease_owner,
6984                        error.to_string(),
6985                        false,
6986                    )
6987                    .await;
6988                if let Err(error) = failure {
6989                    if !activity_task_rejection_is_final(&error) {
6990                        return Err(error);
6991                    }
6992                }
6993            }
6994        }
6995
6996        Ok(ManagedPollOutcome::Handled)
6997    }
6998
6999    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7000        while !stop.load(Ordering::SeqCst) {
7001            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
7002                stop.store(true, Ordering::SeqCst);
7003                break;
7004            }
7005        }
7006
7007        Ok(())
7008    }
7009
7010    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
7011        let poll_request_id = unique_request_id("rust-query-poll");
7012        let response = self
7013            .retry_worker_operation(|| {
7014                self.client.poll_query_task_response_with_request_id(
7015                    &self.worker_id,
7016                    &self.task_queue,
7017                    self.poll_timeout,
7018                    &poll_request_id,
7019                    0,
7020                )
7021            })
7022            .await;
7023        let Some(response) = self.settle_worker_poll_response(response).await? else {
7024            return Ok(ManagedPollOutcome::Idle);
7025        };
7026        if response.outcome().should_stop() {
7027            return Ok(ManagedPollOutcome::Stop);
7028        }
7029        let Some(task) = response.task else {
7030            return Ok(ManagedPollOutcome::Idle);
7031        };
7032
7033        let query_task_id = task.query_task_id.clone();
7034        let attempt = task.query_task_attempt;
7035        let lease_owner = task
7036            .lease_owner
7037            .clone()
7038            .unwrap_or_else(|| self.worker_id.clone());
7039        let codec = task.payload_codec.clone();
7040
7041        match self.execute_query_task(task).await {
7042            Ok(value) => {
7043                let result_envelope = match encode_typed_envelope(&value, &codec) {
7044                    Ok(result_envelope) => result_envelope,
7045                    Err(error) => {
7046                        let failure = self
7047                            .client
7048                            .fail_query_task(
7049                                &query_task_id,
7050                                &lease_owner,
7051                                attempt,
7052                                error.to_string(),
7053                                "query_result_encode_failed",
7054                                "QueryResultEncodeFailed",
7055                            )
7056                            .await;
7057                        if let Err(error) = failure {
7058                            if !query_task_rejection_is_final(&error) {
7059                                return Err(error);
7060                            }
7061                        }
7062                        return Ok(ManagedPollOutcome::Handled);
7063                    }
7064                };
7065
7066                if let Err(error) = self
7067                    .client
7068                    .complete_query_task_with_envelope(
7069                        &query_task_id,
7070                        &lease_owner,
7071                        attempt,
7072                        value.clone().into_json()?,
7073                        result_envelope,
7074                    )
7075                    .await
7076                {
7077                    if !query_task_rejection_is_final(&error) {
7078                        return Err(error);
7079                    }
7080                }
7081            }
7082            Err(failure) => {
7083                let result = self
7084                    .client
7085                    .fail_query_task(
7086                        &query_task_id,
7087                        &lease_owner,
7088                        attempt,
7089                        failure.message,
7090                        failure.reason,
7091                        failure.failure_type,
7092                    )
7093                    .await;
7094                if let Err(error) = result {
7095                    if !query_task_rejection_is_final(&error) {
7096                        return Err(error);
7097                    }
7098                }
7099            }
7100        }
7101
7102        Ok(ManagedPollOutcome::Handled)
7103    }
7104
7105    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7106        while !stop.load(Ordering::SeqCst) {
7107            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
7108                stop.store(true, Ordering::SeqCst);
7109                break;
7110            }
7111        }
7112
7113        Ok(())
7114    }
7115
7116    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7117    where
7118        F: FnMut() -> Fut,
7119        Fut: Future<Output = Result<T>>,
7120    {
7121        let mut retries = 0;
7122
7123        loop {
7124            match operation().await {
7125                Err(error)
7126                    if worker_operation_is_retryable(&error)
7127                        && retries < self.retry_policy.max_retries =>
7128                {
7129                    retries += 1;
7130                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7131                }
7132                result => return result,
7133            }
7134        }
7135    }
7136
7137    async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7138        match response {
7139            Ok(response) => Ok(Some(response)),
7140            Err(error) => {
7141                let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7142                    return Err(error);
7143                };
7144                let minimum_delay = self
7145                    .retry_policy
7146                    .initial_backoff
7147                    .max(Duration::from_millis(1));
7148                let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7149                tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7150                Ok(None)
7151            }
7152        }
7153    }
7154
7155    async fn execute_query_task(
7156        &self,
7157        mut task: QueryTask,
7158    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7159        validate_query_task_payloads(&task).map_err(|error| {
7160            QueryTaskExecutionFailure::new(
7161                "query_payload_decode_failed",
7162                error.to_string(),
7163                "QueryPayloadDecodeFailed",
7164            )
7165        })?;
7166
7167        if !self.workflows.contains_key(&task.workflow_type) {
7168            return Err(QueryTaskExecutionFailure::new(
7169                "query_workflow_type_not_registered",
7170                format!("no workflow registered for type {:?}", task.workflow_type),
7171                "WorkflowTypeNotRegistered",
7172            ));
7173        }
7174
7175        let Some(handlers) = self.queries.get(&task.workflow_type) else {
7176            return Err(QueryTaskExecutionFailure::new(
7177                "query_handler_unavailable",
7178                format!(
7179                    "query handlers are unavailable for workflow type {:?}",
7180                    task.workflow_type
7181                ),
7182                "QueryHandlerUnavailable",
7183            ));
7184        };
7185        let Some(query) = handlers.get(&task.query_name) else {
7186            return Err(QueryTaskExecutionFailure::new(
7187                "rejected_unknown_query",
7188                format!("unknown query {:?}", task.query_name),
7189                "QueryFailed",
7190            ));
7191        };
7192
7193        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7194            .map_err(|error| {
7195            QueryTaskExecutionFailure::new(
7196                "query_payload_decode_failed",
7197                format!("cannot decode query arguments: {error}"),
7198                "QueryPayloadDecodeFailed",
7199            )
7200        })?;
7201        let workflow_input_typed =
7202            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7203                .map_err(|error| {
7204                    QueryTaskExecutionFailure::new(
7205                        "query_workflow_state_unavailable",
7206                        format!("cannot decode workflow start input: {error}"),
7207                        "QueryWorkflowStateUnavailable",
7208                    )
7209                })?;
7210        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7211            QueryTaskExecutionFailure::new(
7212                "query_workflow_state_unavailable",
7213                format!("cannot project workflow start input: {error}"),
7214                "QueryWorkflowStateUnavailable",
7215            )
7216        })?;
7217        hydrate_query_history_from_export(&mut task).map_err(|error| {
7218            QueryTaskExecutionFailure::new(
7219                "query_workflow_state_unavailable",
7220                format!("cannot restore query history snapshot: {error}"),
7221                "QueryWorkflowStateUnavailable",
7222            )
7223        })?;
7224        enrich_query_history_from_export(&mut task).map_err(|error| {
7225            QueryTaskExecutionFailure::new(
7226                "query_workflow_state_unavailable",
7227                format!("cannot restore compact query history payloads: {error}"),
7228                "QueryWorkflowStateUnavailable",
7229            )
7230        })?;
7231        let signal_events = query_signal_events(&task).map_err(|error| {
7232            QueryTaskExecutionFailure::new(
7233                "query_workflow_state_unavailable",
7234                format!("cannot decode committed workflow signals: {error}"),
7235                "QueryWorkflowStateUnavailable",
7236            )
7237        })?;
7238        let history_events = Arc::new(std::mem::take(&mut task.history_events));
7239        let context = QueryContext {
7240            workflow_id: task.workflow_id,
7241            run_id: task.run_id,
7242            workflow_type: task.workflow_type.clone(),
7243            run_status: task.run_status,
7244            workflow_input,
7245            workflow_input_avro_value: workflow_input_typed.clone(),
7246            history_events: Arc::clone(&history_events),
7247            signal_events: Arc::new(signal_events),
7248        };
7249
7250        let future = match query {
7251            RegisteredQuery::Snapshot(handler) => handler(context, args),
7252            RegisteredQuery::Replayed {
7253                state_type,
7254                handler,
7255            } => {
7256                let workflow = self
7257                    .workflows
7258                    .get(&task.workflow_type)
7259                    .expect("workflow registration was checked above");
7260                if workflow.state_type != Some(*state_type) {
7261                    return Err(QueryTaskExecutionFailure::new(
7262                        "query_workflow_state_unavailable",
7263                        "replayed query state type does not match its workflow registration",
7264                        "QueryWorkflowStateUnavailable",
7265                    ));
7266                }
7267                let replay = workflow.replay.as_ref().ok_or_else(|| {
7268                    QueryTaskExecutionFailure::new(
7269                        "query_workflow_state_unavailable",
7270                        format!(
7271                            "workflow type {:?} is not registered for instance-state replay",
7272                            task.workflow_type
7273                        ),
7274                        "QueryWorkflowStateUnavailable",
7275                    )
7276                })?;
7277                let workflow_state = Arc::new(Mutex::new(
7278                    WorkflowState::new_with_identity(
7279                        history_events.as_ref().clone(),
7280                        context.workflow_id.clone(),
7281                        context.run_id.clone(),
7282                        self.task_queue.clone(),
7283                        task.payload_codec,
7284                        None,
7285                    )
7286                    .map_err(|error| {
7287                        QueryTaskExecutionFailure::new(
7288                            "query_workflow_state_unavailable",
7289                            format!("workflow replay failed before query: {error}"),
7290                            "QueryWorkflowStateUnavailable",
7291                        )
7292                    })?,
7293                ));
7294                let workflow_context = WorkflowContext {
7295                    state: workflow_state,
7296                };
7297                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7298                let mut cx = TaskContext::from_waker(noop_waker_ref());
7299                match invocation.future.as_mut().poll(&mut cx) {
7300                    Poll::Ready(Ok(_)) => {
7301                        workflow_context
7302                            .ensure_history_consumed()
7303                            .map_err(|error| {
7304                                QueryTaskExecutionFailure::new(
7305                                    "query_workflow_state_unavailable",
7306                                    format!("workflow replay failed before query: {error}"),
7307                                    "QueryWorkflowStateUnavailable",
7308                                )
7309                            })?;
7310                    }
7311                    Poll::Ready(Err(error)) => {
7312                        return Err(QueryTaskExecutionFailure::new(
7313                            "query_workflow_state_unavailable",
7314                            format!("workflow replay failed before query: {error}"),
7315                            "QueryWorkflowStateUnavailable",
7316                        ));
7317                    }
7318                    Poll::Pending => {
7319                        let commands = workflow_context.take_commands().map_err(|error| {
7320                            QueryTaskExecutionFailure::new(
7321                                "query_workflow_state_unavailable",
7322                                format!("workflow replay failed before query: {error}"),
7323                                "QueryWorkflowStateUnavailable",
7324                            )
7325                        })?;
7326                        if commands.is_empty()
7327                            && !workflow_context
7328                                .matched_recorded_pending()
7329                                .map_err(|error| {
7330                                    QueryTaskExecutionFailure::new(
7331                                        "query_workflow_state_unavailable",
7332                                        format!("workflow replay failed before query: {error}"),
7333                                        "QueryWorkflowStateUnavailable",
7334                                    )
7335                                })?
7336                        {
7337                            return Err(QueryTaskExecutionFailure::new(
7338                                "query_workflow_state_unavailable",
7339                                "workflow replay yielded without a durable command",
7340                                "QueryWorkflowStateUnavailable",
7341                            ));
7342                        }
7343                    }
7344                }
7345                let state = (invocation.snapshot)().map_err(|error| {
7346                    QueryTaskExecutionFailure::new(
7347                        "query_workflow_state_unavailable",
7348                        format!("cannot snapshot replayed workflow state: {error}"),
7349                        "QueryWorkflowStateUnavailable",
7350                    )
7351                })?;
7352                handler(context, state, args).map_err(|message| {
7353                    QueryTaskExecutionFailure::new(
7354                        "query_workflow_state_unavailable",
7355                        message,
7356                        "QueryWorkflowStateUnavailable",
7357                    )
7358                })?
7359            }
7360        };
7361
7362        future.await.map_err(|error| {
7363            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7364        })
7365    }
7366
7367    #[cfg(test)]
7368    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7369        Ok(self.execute_workflow_task_decision(task)?.commands)
7370    }
7371
7372    fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7373        validate_workflow_task_payloads(&task)?;
7374
7375        if let Some(update_id) = task
7376            .workflow_update_id
7377            .as_deref()
7378            .filter(|update_id| !update_id.is_empty())
7379        {
7380            return self
7381                .execute_update_task(&task, update_id)
7382                .map(WorkflowTaskDecision::without_message_streams);
7383        }
7384
7385        let workflow = self
7386            .workflows
7387            .get(&task.workflow_type)
7388            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7389        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7390        let resume_signal = decode_resume_signal(&task)?;
7391        let history_budget = WorkflowHistoryBudget {
7392            event_count: task
7393                .total_history_events
7394                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7395            size_bytes: task.history_size_bytes,
7396            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7397            pressure: task.history_budget_pressure.clone(),
7398        };
7399        let workflow_command_identity = task
7400            .workflow_command_id
7401            .clone()
7402            .filter(|identity| !identity.is_empty())
7403            .unwrap_or_default();
7404        let mut workflow_state = WorkflowState::new_with_identity(
7405            task.history_events,
7406            task.workflow_id,
7407            task.run_id,
7408            self.task_queue.clone(),
7409            task.payload_codec.clone(),
7410            resume_signal,
7411        )?;
7412        workflow_state.history_budget = history_budget;
7413        workflow_state.workflow_command_identity = workflow_command_identity;
7414        workflow_state.cancel_requested = task.cancel_requested;
7415        let state = Arc::new(Mutex::new(workflow_state));
7416        let ctx = WorkflowContext { state };
7417        let mut future = (workflow.execute)(ctx.clone(), input);
7418        let mut cx = TaskContext::from_waker(noop_waker_ref());
7419
7420        match future.as_mut().poll(&mut cx) {
7421            Poll::Ready(Ok(result)) => {
7422                ctx.ensure_history_consumed()?;
7423                let result = encode_typed_envelope(&result, &task.payload_codec)?;
7424                let mut commands = ctx.take_commands()?;
7425                commands.push(json!({
7426                    "type": "complete_workflow",
7427                    "result": result
7428                }));
7429                self.message_stream_decision(&ctx, commands)
7430            }
7431            Poll::Ready(Err(error)) => {
7432                if let Error::ContinueAsNew(request) = error {
7433                    let mut commands = ctx.take_commands()?;
7434                    if let Some(command) = ctx.continue_as_new_command(request)? {
7435                        commands.push(command);
7436                    }
7437                    ctx.ensure_history_consumed()?;
7438                    return self.message_stream_decision(&ctx, commands);
7439                }
7440                if workflow_task_integrity_error(&error) {
7441                    // Replay and protocol failures describe the workflow-task
7442                    // decision itself. Preserve their specific failure reason
7443                    // instead of replacing it with the derivative fact that
7444                    // recorded commands remain unconsumed.
7445                    return Err(error);
7446                }
7447                // A handler error must not hide a committed durable command that
7448                // upgraded workflow code no longer consumes.
7449                ctx.ensure_history_consumed()?;
7450                let mut commands = ctx.take_commands()?;
7451                commands.push(workflow_failure_command(&error));
7452                self.message_stream_decision(&ctx, commands)
7453            }
7454            Poll::Pending => {
7455                let commands = ctx.take_commands()?;
7456                if commands.is_empty() && !ctx.matched_recorded_pending()? {
7457                    Err(Error::WorkflowYieldedWithoutCommand)
7458                } else {
7459                    self.message_stream_decision(&ctx, commands)
7460                }
7461            }
7462        }
7463    }
7464
7465    fn message_stream_decision(
7466        &self,
7467        ctx: &WorkflowContext,
7468        commands: Vec<Value>,
7469    ) -> Result<WorkflowTaskDecision> {
7470        let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7471        Ok(WorkflowTaskDecision {
7472            commands,
7473            message_stream_cursors,
7474            message_stream_waits,
7475        })
7476    }
7477
7478    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7479        if !self.workflows.contains_key(&task.workflow_type) {
7480            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7481        }
7482
7483        let accepted = task.history_events.iter().rev().find_map(|event| {
7484            (event.event_type == "UpdateAccepted"
7485                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7486            .then_some(&event.payload)
7487        });
7488        let update_name = accepted
7489            .and_then(|payload| payload.get("update_name"))
7490            .and_then(Value::as_str)
7491            .or(task.update_name.as_deref())
7492            .unwrap_or_default();
7493        let Some(handler) = self
7494            .updates
7495            .get(&task.workflow_type)
7496            .and_then(|handlers| handlers.get(update_name))
7497        else {
7498            return Ok(vec![json!({
7499                "type": "fail_update",
7500                "update_id": update_id,
7501                "message": format!(
7502                    "no update handler is registered for {}.{update_name}",
7503                    task.workflow_type
7504                ),
7505                "exception_type": "UnknownUpdate",
7506                "non_retryable": true,
7507            })]);
7508        };
7509        let arguments = accepted
7510            .and_then(|payload| payload.get("arguments"))
7511            .or(task.arguments.as_ref());
7512        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7513        let context = QueryContext {
7514            workflow_id: task.workflow_id.clone(),
7515            run_id: task.run_id.clone(),
7516            workflow_type: task.workflow_type.clone(),
7517            run_status: Some("running".to_string()),
7518            workflow_input: Value::Null,
7519            workflow_input_avro_value: AvroValue::Null,
7520            history_events: Arc::new(task.history_events.clone()),
7521            signal_events: Arc::new(Vec::new()),
7522        };
7523        let mut future = handler(context, arguments);
7524        let mut cx = TaskContext::from_waker(noop_waker_ref());
7525
7526        match future.as_mut().poll(&mut cx) {
7527            Poll::Ready(Ok(result)) => Ok(vec![json!({
7528                "type": "complete_update",
7529                "update_id": update_id,
7530                "result": encode_typed_envelope(&result, &task.payload_codec)?,
7531            })]),
7532            Poll::Ready(Err(error)) => Ok(vec![json!({
7533                "type": "fail_update",
7534                "update_id": update_id,
7535                "message": error.to_string(),
7536                "exception_type": "UpdateFailed",
7537                "non_retryable": true,
7538            })]),
7539            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7540        }
7541    }
7542
7543    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7544        validate_activity_task_payloads(&task)?;
7545
7546        let handler = self
7547            .activities
7548            .get(&task.activity_type)
7549            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7550        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7551        let attempt_id = task
7552            .activity_attempt_id
7553            .clone()
7554            .or(task.attempt_id.clone())
7555            .unwrap_or_default();
7556        let lease_owner = task
7557            .lease_owner
7558            .clone()
7559            .unwrap_or_else(|| self.worker_id.clone());
7560        let ctx = ActivityContext {
7561            client: self.client.clone(),
7562            task_id: task.task_id,
7563            activity_attempt_id: attempt_id,
7564            lease_owner,
7565            activity_type: task.activity_type,
7566            attempt_number: task.attempt_number,
7567            task_queue: self.task_queue.clone(),
7568            worker_id: self.worker_id.clone(),
7569        };
7570
7571        handler(ctx, args).await
7572    }
7573}
7574
7575fn poller_result(
7576    kind: &str,
7577    result: std::result::Result<Result<()>, tokio::task::JoinError>,
7578) -> Result<()> {
7579    match result {
7580        Ok(result) => result,
7581        Err(error) => Err(Error::WorkerLoop(format!(
7582            "{kind} poller join error: {error}"
7583        ))),
7584    }
7585}
7586
7587fn optional_poller_result(
7588    kind: &str,
7589    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7590) -> Result<()> {
7591    match result {
7592        Some(result) => poller_result(kind, result),
7593        None => Ok(()),
7594    }
7595}
7596
7597async fn join_pollers(
7598    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7599    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7600    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7601) -> Result<()> {
7602    let mut first_error = None;
7603
7604    if let Some(handle) = workflow_poller {
7605        if let Err(error) = poller_result("workflow", handle.await) {
7606            first_error.get_or_insert(error);
7607        }
7608    }
7609
7610    if let Some(handle) = activity_poller {
7611        if let Err(error) = poller_result("activity", handle.await) {
7612            first_error.get_or_insert(error);
7613        }
7614    }
7615
7616    if let Some(handle) = query_poller {
7617        if let Err(error) = poller_result("query", handle.await) {
7618            first_error.get_or_insert(error);
7619        }
7620    }
7621
7622    if let Some(error) = first_error {
7623        Err(error)
7624    } else {
7625        Ok(())
7626    }
7627}
7628
7629fn default_worker_id() -> String {
7630    let millis = SystemTime::now()
7631        .duration_since(UNIX_EPOCH)
7632        .unwrap_or_default()
7633        .as_millis();
7634    format!("rust-worker-{}-{millis}", std::process::id())
7635}
7636
7637fn percent_encode_path_segment(segment: &str) -> String {
7638    const HEX: &[u8; 16] = b"0123456789ABCDEF";
7639    let mut encoded = String::with_capacity(segment.len());
7640
7641    for byte in segment.bytes() {
7642        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7643            encoded.push(char::from(byte));
7644        } else {
7645            encoded.push('%');
7646            encoded.push(char::from(HEX[(byte >> 4) as usize]));
7647            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7648        }
7649    }
7650
7651    encoded
7652}
7653
7654fn unique_request_id(prefix: &str) -> String {
7655    let nanos = SystemTime::now()
7656        .duration_since(UNIX_EPOCH)
7657        .unwrap_or_default()
7658        .as_nanos();
7659    format!("{prefix}-{}-{nanos}", std::process::id())
7660}
7661
7662#[derive(Debug)]
7663struct QueryTaskExecutionFailure {
7664    reason: String,
7665    message: String,
7666    failure_type: String,
7667}
7668
7669impl QueryTaskExecutionFailure {
7670    fn new(
7671        reason: impl Into<String>,
7672        message: impl Into<String>,
7673        failure_type: impl Into<String>,
7674    ) -> Self {
7675        Self {
7676            reason: reason.into(),
7677            message: message.into(),
7678            failure_type: failure_type.into(),
7679        }
7680    }
7681}
7682
7683/// Typed local state owned by one deterministic workflow invocation.
7684///
7685/// Use [`WorkflowInstance::update`] for the same state transitions during
7686/// ordinary execution and replay. A replayed query receives a detached
7687/// immutable `Arc<S>` rather than this mutation-capable handle.
7688#[derive(Clone, Debug)]
7689pub struct WorkflowInstance<S> {
7690    state: Arc<Mutex<S>>,
7691}
7692
7693impl<S> WorkflowInstance<S> {
7694    fn new(state: S) -> Self {
7695        Self {
7696            state: Arc::new(Mutex::new(state)),
7697        }
7698    }
7699
7700    /// Read the current workflow-instance state without changing it.
7701    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7702        let state = self
7703            .state
7704            .lock()
7705            .map_err(|_| Error::WorkflowStatePoisoned)?;
7706        Ok(reader(&state))
7707    }
7708
7709    /// Apply one deterministic workflow-instance state transition.
7710    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7711        let mut state = self
7712            .state
7713            .lock()
7714            .map_err(|_| Error::WorkflowStatePoisoned)?;
7715        Ok(transition(&mut state))
7716    }
7717}
7718
7719impl<S: Clone> WorkflowInstance<S> {
7720    fn snapshot(&self) -> Result<S> {
7721        self.read(Clone::clone)
7722    }
7723}
7724
7725#[derive(Clone, Debug, PartialEq)]
7726pub struct MessageStreamMessage {
7727    pub stream_name: String,
7728    pub message_id: String,
7729    pub position: u64,
7730    pub arguments: Vec<AvroValue>,
7731}
7732
7733#[derive(Clone, Debug)]
7734pub struct MessageStream {
7735    ctx: WorkflowContext,
7736    name: String,
7737}
7738
7739impl MessageStream {
7740    /// Wait for one message, then return a bounded currently-available batch.
7741    pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7742        if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7743            return Err(Error::Codec(format!(
7744                "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7745            )));
7746        }
7747        loop {
7748            if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7749                return Ok(batch);
7750            }
7751
7752            self.ctx.record_message_stream_wait(&self.name)?;
7753            let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7754            let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7755            self.ctx.buffer_message_stream_delivery(arguments)?;
7756            if let Some(sequence) = replay_wait_sequence {
7757                self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7758            }
7759        }
7760    }
7761
7762    pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7763        self.receive(1)
7764            .await?
7765            .into_iter()
7766            .next()
7767            .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7768    }
7769}
7770
7771#[derive(Clone, Debug)]
7772pub struct WorkflowContext {
7773    state: Arc<Mutex<WorkflowState>>,
7774}
7775
7776fn valid_memo_key(key: &str) -> bool {
7777    let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7778
7779    !key.is_empty()
7780        && key.len() <= 64
7781        && (numeric_candidate.is_empty()
7782            || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7783        && key
7784            .bytes()
7785            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7786}
7787
7788fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7789    BASE64
7790        .decode(encode_avro_value(value)?.blob)
7791        .map(|bytes| bytes.len())
7792        .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7793}
7794
7795fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7796    let AvroValue::Map(entries) = value else {
7797        return Err(Error::InvalidMemoUpdate(
7798            "entries must serialize to an Avro string-keyed map".to_string(),
7799        ));
7800    };
7801    if require_entries && entries.is_empty() {
7802        return Err(Error::InvalidMemoUpdate(
7803            "at least one entry is required".to_string(),
7804        ));
7805    }
7806    if entries.len() > MAX_MEMO_ENTRIES {
7807        return Err(Error::InvalidMemoUpdate(format!(
7808            "at most {MAX_MEMO_ENTRIES} entries are allowed"
7809        )));
7810    }
7811
7812    for (key, value) in &entries {
7813        if !valid_memo_key(&key) {
7814            return Err(Error::InvalidMemoUpdate(
7815                "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7816            ));
7817        }
7818        if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7819            return Err(Error::InvalidMemoUpdate(format!(
7820                "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7821            )));
7822        }
7823    }
7824
7825    let value = AvroValue::Map(entries);
7826    if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7827        return Err(Error::InvalidMemoUpdate(format!(
7828            "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7829        )));
7830    }
7831    Ok(value)
7832}
7833
7834fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7835    let object = envelope.as_object().ok_or_else(|| {
7836        Error::InvalidMemoUpdate(
7837            "history field must use the public {codec, blob} payload envelope".to_string(),
7838        )
7839    })?;
7840    if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7841        return Err(Error::InvalidMemoUpdate(
7842            "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7843        ));
7844    }
7845
7846    canonical_memo_entries(
7847        decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7848        require_entries,
7849    )
7850}
7851
7852impl WorkflowContext {
7853    pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7854        let name = name.into();
7855        if name.is_empty()
7856            || name.len() > 128
7857            || !name.bytes().all(|byte| {
7858                byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7859            })
7860        {
7861            return Err(Error::Codec(
7862                "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7863                    .to_string(),
7864            ));
7865        }
7866        Ok(MessageStream {
7867            ctx: self.clone(),
7868            name,
7869        })
7870    }
7871
7872    fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7873        let mut state = self
7874            .state
7875            .lock()
7876            .map_err(|_| Error::WorkflowStatePoisoned)?;
7877        let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7878        state
7879            .message_stream_waits
7880            .insert(name.to_string(), position);
7881        Ok(())
7882    }
7883
7884    fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7885        let mut state = self
7886            .state
7887            .lock()
7888            .map_err(|_| Error::WorkflowStatePoisoned)?;
7889        let cursor = state
7890            .message_stream_cursors
7891            .get(&message.stream_name)
7892            .copied()
7893            .unwrap_or(0);
7894        if message.position <= cursor {
7895            return Ok(());
7896        }
7897        let pending = state
7898            .message_stream_messages
7899            .entry(message.stream_name.clone())
7900            .or_default();
7901        if pending.iter().any(|candidate| {
7902            candidate.position == message.position || candidate.message_id == message.message_id
7903        }) {
7904            return Ok(());
7905        }
7906        pending.push(message);
7907        pending.sort_by_key(|candidate| candidate.position);
7908        Ok(())
7909    }
7910
7911    fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7912        if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7913            match delivery {
7914                MessageStreamDelivery::Message(message) => {
7915                    let stream_name = message.stream_name.clone();
7916                    self.buffer_message_stream(message)?;
7917                    return Ok(Some(stream_name));
7918                }
7919                MessageStreamDelivery::Cursor {
7920                    stream_name,
7921                    through_position,
7922                } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7923            }
7924        }
7925        Ok(None)
7926    }
7927
7928    fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7929        let state = self
7930            .state
7931            .lock()
7932            .map_err(|_| Error::WorkflowStatePoisoned)?;
7933        Ok(match state.recorded_commands.get(state.command_cursor) {
7934            Some(RecordedCommand::SignalWait {
7935                sequence,
7936                signal_name,
7937                ..
7938            }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7939            _ => None,
7940        })
7941    }
7942
7943    fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7944        let (history, payload_codec) = {
7945            let state = self
7946                .state
7947                .lock()
7948                .map_err(|_| Error::WorkflowStatePoisoned)?;
7949            (
7950                Arc::clone(&state.history_events),
7951                state.payload_codec.clone(),
7952            )
7953        };
7954
7955        let Some(opened_index) = history.iter().position(|event| {
7956            event.event_type == "SignalWaitOpened"
7957                && durable_event_sequence(event) == Some(wait_sequence)
7958                && event.payload.get("signal_name").and_then(Value::as_str)
7959                    == Some(MESSAGE_STREAM_SIGNAL)
7960        }) else {
7961            return Ok(());
7962        };
7963        let boundary_index = history
7964            .iter()
7965            .enumerate()
7966            .skip(opened_index + 1)
7967            .find_map(|(index, event)| {
7968                (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7969                    && is_authored_command_open_event(event))
7970                .then_some(index)
7971            })
7972            .unwrap_or(history.len());
7973
7974        for event in history[opened_index + 1..boundary_index]
7975            .iter()
7976            .filter(|event| {
7977                event.event_type == "SignalReceived"
7978                    && event.payload.get("signal_name").and_then(Value::as_str)
7979                        == Some(MESSAGE_STREAM_SIGNAL)
7980            })
7981        {
7982            let arguments = decode_signal_event_arguments(event, &payload_codec)?
7983                .into_iter()
7984                .map(AvroValue::into_json)
7985                .collect::<Result<Vec<_>>>()?;
7986            self.buffer_message_stream_delivery(arguments)?;
7987        }
7988        Ok(())
7989    }
7990
7991    fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7992        let mut state = self
7993            .state
7994            .lock()
7995            .map_err(|_| Error::WorkflowStatePoisoned)?;
7996        let cursor = state
7997            .message_stream_cursors
7998            .entry(name.to_string())
7999            .or_default();
8000        *cursor = (*cursor).max(through_position);
8001        if let Some(pending) = state.message_stream_messages.get_mut(name) {
8002            pending.retain(|message| message.position > through_position);
8003        }
8004        Ok(())
8005    }
8006
8007    fn take_message_stream_batch(
8008        &self,
8009        name: &str,
8010        max_items: usize,
8011    ) -> Result<Option<Vec<MessageStreamMessage>>> {
8012        let mut state = self
8013            .state
8014            .lock()
8015            .map_err(|_| Error::WorkflowStatePoisoned)?;
8016        let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
8017        let pending = state
8018            .message_stream_messages
8019            .entry(name.to_string())
8020            .or_default();
8021        let count = contiguous_message_stream_count(pending, cursor, max_items);
8022        if count == 0 {
8023            return Ok(None);
8024        }
8025        let batch = pending.drain(..count).collect::<Vec<_>>();
8026        let position = batch.last().map(|message| message.position).unwrap_or(0);
8027        state
8028            .message_stream_cursors
8029            .insert(name.to_string(), position);
8030        state.message_stream_waits.remove(name);
8031        Ok(Some(batch))
8032    }
8033
8034    fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
8035        let state = self
8036            .state
8037            .lock()
8038            .map_err(|_| Error::WorkflowStatePoisoned)?;
8039        let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
8040        cursors.sort_by_key(|(name, _)| *name);
8041        let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
8042        waits.sort_by_key(|(name, _)| *name);
8043        Ok((
8044            cursors
8045                .into_iter()
8046                .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
8047                .collect(),
8048            waits
8049                .into_iter()
8050                .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
8051                .collect(),
8052        ))
8053    }
8054    /// Identity of the parent workflow currently being replayed.
8055    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
8056        let state = self
8057            .state
8058            .lock()
8059            .map_err(|_| Error::WorkflowStatePoisoned)?;
8060        Ok(WorkflowIdentity {
8061            workflow_id: state.workflow_id.clone(),
8062            run_id: state.run_id.clone(),
8063        })
8064    }
8065
8066    /// Return the server-published history budget for this workflow task.
8067    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
8068        let state = self
8069            .state
8070            .lock()
8071            .map_err(|_| Error::WorkflowStatePoisoned)?;
8072        Ok(state.history_budget.clone())
8073    }
8074
8075    /// Continue this workflow instance as a fresh run with replacement arguments.
8076    ///
8077    /// Return this value directly from the workflow handler. The worker converts
8078    /// it to the terminal protocol command only after replay has consumed every
8079    /// recorded durable command.
8080    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
8081        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
8082    }
8083
8084    /// Continue as new with optional workflow-type and task-queue overrides.
8085    pub fn continue_as_new_with_options<T: Serialize>(
8086        &self,
8087        options: ContinueAsNewOptions,
8088        args: T,
8089    ) -> Result<Value> {
8090        options.validate()?;
8091        Err(Error::ContinueAsNew(ContinueAsNewRequest {
8092            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
8093            options,
8094        }))
8095    }
8096
8097    pub fn activity<T: Serialize>(
8098        &self,
8099        activity_type: impl Into<String>,
8100        args: T,
8101    ) -> ActivityCall {
8102        self.activity_with_options(activity_type, ActivityOptions::new(), args)
8103    }
8104
8105    pub fn activity_on_queue<T, Q>(
8106        &self,
8107        activity_type: impl Into<String>,
8108        task_queue: Option<Q>,
8109        args: T,
8110    ) -> ActivityCall
8111    where
8112        T: Serialize,
8113        Q: Into<String>,
8114    {
8115        let mut options = ActivityOptions::new();
8116        options.task_queue = task_queue.map(Into::into);
8117        self.activity_with_options(activity_type, options, args)
8118    }
8119
8120    /// Schedule one durable activity with retry, routing, and timeout options.
8121    ///
8122    /// Options are validated before the command is emitted. Once the command is
8123    /// recorded, replay consumes the same activity lifecycle at this command
8124    /// position and never emits a duplicate schedule.
8125    ///
8126    /// ```no_run
8127    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
8128    /// # use std::time::Duration;
8129    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
8130    /// let result = ctx
8131    ///     .activity_with_options(
8132    ///         "charge-card",
8133    ///         ActivityOptions::new()
8134    ///             .task_queue("payments")
8135    ///             .retry_policy(
8136    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
8137    ///                     Duration::from_secs(1),
8138    ///                     2,
8139    ///                     Some(Duration::from_secs(30)),
8140    ///                 ),
8141    ///             )
8142    ///             .start_to_close_timeout(Duration::from_secs(60))
8143    ///             .schedule_to_close_timeout(Duration::from_secs(180))
8144    ///             .heartbeat_timeout(Duration::from_secs(15)),
8145    ///         json!([{"order_id": "order-42"}]),
8146    ///     )
8147    ///     .await;
8148    /// match result {
8149    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
8150    ///         "reason": failure.reason,
8151    ///         "timeout_kind": failure.timeout_kind,
8152    ///     })),
8153    ///     other => other,
8154    /// }
8155    /// # }
8156    /// ```
8157    pub fn activity_with_options<T: Serialize>(
8158        &self,
8159        activity_type: impl Into<String>,
8160        options: ActivityOptions,
8161        args: T,
8162    ) -> ActivityCall {
8163        ActivityCall {
8164            ctx: self.clone(),
8165            activity_type: activity_type.into(),
8166            options,
8167            args: Some(AvroValue::from_serialize(&args)),
8168            scheduled: false,
8169            parallel_group_path: Vec::new(),
8170        }
8171    }
8172
8173    pub async fn activity_avro_value<T: Serialize>(
8174        &self,
8175        activity_type: impl Into<String>,
8176        args: T,
8177    ) -> Result<AvroValue> {
8178        let mut call = self.activity(activity_type, args);
8179        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8180    }
8181
8182    pub async fn activity_avro_value_with_options<T: Serialize>(
8183        &self,
8184        activity_type: impl Into<String>,
8185        options: ActivityOptions,
8186        args: T,
8187    ) -> Result<AvroValue> {
8188        let mut call = self.activity_with_options(activity_type, options, args);
8189        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8190    }
8191
8192    /// Schedule an activity with a Serde request and decode its Serde result.
8193    pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8194    where
8195        I: Serialize,
8196        O: DeserializeOwned,
8197    {
8198        self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8199            .await
8200    }
8201
8202    /// Schedule an activity with options and decode its result into `O`.
8203    ///
8204    /// Both directions use the fixed Avro Value codec. In particular, this
8205    /// method does not deserialize the JSON-safe inspection projection returned
8206    /// by the dynamic [`ActivityCall`] future.
8207    pub async fn activity_typed_with_options<I, O>(
8208        &self,
8209        activity_type: impl Into<String>,
8210        options: ActivityOptions,
8211        args: I,
8212    ) -> Result<O>
8213    where
8214        I: Serialize,
8215        O: DeserializeOwned,
8216    {
8217        let activity_type = activity_type.into();
8218        let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8219            handler_type_error::<I>(
8220                HandlerKind::Activity,
8221                &activity_type,
8222                HandlerValueKind::Input,
8223                error.to_string(),
8224            )
8225        });
8226        let mut call = ActivityCall {
8227            ctx: self.clone(),
8228            activity_type: activity_type.clone(),
8229            options,
8230            args: Some(encoded),
8231            scheduled: false,
8232            parallel_group_path: Vec::new(),
8233        };
8234        let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8235        decode_handler_result(result, HandlerKind::Activity, &activity_type)
8236    }
8237
8238    /// Schedule and join a deterministic activity/child/timer group.
8239    ///
8240    /// Nested groups retain their input shape. Every durable leaf is scheduled
8241    /// before this future yields, results are assembled by declaration order,
8242    /// and a failure returns [`Error::ParallelFailed`] with typed cause,
8243    /// declaration path, stable group metadata, and completed siblings.
8244    pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8245        ParallelCall::new(self.clone(), operations)
8246    }
8247
8248    /// Alias for [`WorkflowContext::parallel`].
8249    pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8250        self.parallel(operations)
8251    }
8252
8253    /// Lossless fixed-Avro variant of [`WorkflowContext::parallel`].
8254    pub async fn parallel_avro_value(
8255        &self,
8256        operations: Vec<ParallelOperation>,
8257    ) -> Result<Vec<ParallelAvroResult>> {
8258        let mut call = self.parallel(operations);
8259        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8260    }
8261
8262    /// Start every durable operation and resume from the one winner persisted
8263    /// by Server. Non-winning operations continue and remain addressable.
8264    pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8265        let operations = operations
8266            .into_iter()
8267            .enumerate()
8268            .map(|(index, operation)| (SelectionKey::Index(index), operation))
8269            .collect();
8270        SelectCall::new(self.clone(), operations)
8271    }
8272
8273    /// Named-key variant of [`WorkflowContext::select`].
8274    pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8275    where
8276        K: Into<SelectionKey>,
8277    {
8278        SelectCall::new(
8279            self.clone(),
8280            operations
8281                .into_iter()
8282                .map(|(key, operation)| (key.into(), operation))
8283                .collect(),
8284        )
8285    }
8286
8287    /// Create a workflow-local deterministic compensation registry.
8288    pub fn saga(&self) -> Saga {
8289        Saga::new(self.clone())
8290    }
8291
8292    /// Whether the current workflow task carries a cooperative cancel request.
8293    pub fn is_cancellation_requested(&self) -> Result<bool> {
8294        let state = self
8295            .state
8296            .lock()
8297            .map_err(|_| Error::WorkflowStatePoisoned)?;
8298        Ok(state.cancel_requested)
8299    }
8300
8301    /// Raise a typed cooperative cancellation at an author-controlled point.
8302    ///
8303    /// Passing this result to [`Saga::finish`] compensates already registered
8304    /// forward steps before the cancellation remains the initiating outcome.
8305    pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8306        if self.is_cancellation_requested()? {
8307            return Err(Error::WorkflowCancellationRequested(
8308                WorkflowCancellationRequested,
8309            ));
8310        }
8311        Ok(())
8312    }
8313
8314    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8315        SignalCall {
8316            ctx: self.clone(),
8317            signal_name: signal_name.into(),
8318            runtime_reserved_allowed: false,
8319            opened_wait: false,
8320            matched_pending: false,
8321            parallel_group_path: Vec::new(),
8322        }
8323    }
8324
8325    fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8326        SignalCall {
8327            ctx: self.clone(),
8328            signal_name: signal_name.into(),
8329            runtime_reserved_allowed: true,
8330            opened_wait: false,
8331            matched_pending: false,
8332            parallel_group_path: Vec::new(),
8333        }
8334    }
8335
8336    pub async fn wait_signal_avro_value(
8337        &self,
8338        signal_name: impl Into<String>,
8339    ) -> Result<Vec<AvroValue>> {
8340        let mut call = self.wait_signal(signal_name);
8341        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8342    }
8343
8344    /// Return every committed signal argument list with the given name.
8345    ///
8346    /// This history-backed view is deterministic and is intended for
8347    /// condition predicates that must be re-evaluated after a signal while the
8348    /// workflow is blocked on [`WorkflowContext::wait_condition`].
8349    pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8350        self.signals_avro_value(signal_name)?
8351            .into_iter()
8352            .map(|arguments| {
8353                arguments
8354                    .into_iter()
8355                    .map(AvroValue::into_json)
8356                    .collect::<Result<Vec<_>>>()
8357            })
8358            .collect()
8359    }
8360
8361    /// Lossless fixed Avro Value view of committed signals with the given name.
8362    pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8363        let state = self
8364            .state
8365            .lock()
8366            .map_err(|_| Error::WorkflowStatePoisoned)?;
8367        state
8368            .history_events
8369            .iter()
8370            .filter(|event| {
8371                event.event_type == "SignalReceived"
8372                    && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8373            })
8374            .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8375            .collect()
8376    }
8377
8378    /// Return every committed update argument list with the given name.
8379    ///
8380    /// Accepted and applied records for the same update ID are de-duplicated.
8381    /// A Server task created after an update therefore replays the workflow and
8382    /// re-evaluates an open condition without application polling.
8383    pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8384        self.updates_avro_value(update_name)?
8385            .into_iter()
8386            .map(|arguments| {
8387                arguments
8388                    .into_iter()
8389                    .map(AvroValue::into_json)
8390                    .collect::<Result<Vec<_>>>()
8391            })
8392            .collect()
8393    }
8394
8395    /// Lossless fixed Avro Value view of committed updates with the given name.
8396    pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8397        let state = self
8398            .state
8399            .lock()
8400            .map_err(|_| Error::WorkflowStatePoisoned)?;
8401        let mut seen = Vec::new();
8402        let mut updates = Vec::new();
8403        for event in state.history_events.iter() {
8404            if !matches!(
8405                event.event_type.as_str(),
8406                "UpdateAccepted" | "UpdateApplied"
8407            ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8408                || event.payload.get("arguments").is_none()
8409            {
8410                continue;
8411            }
8412            if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8413                if seen.iter().any(|recorded| recorded == update_id) {
8414                    continue;
8415                }
8416                seen.push(update_id.to_string());
8417            }
8418            updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8419        }
8420        Ok(updates)
8421    }
8422
8423    /// Wait for a deterministic predicate to become true or for its durable
8424    /// timeout to elapse.
8425    ///
8426    /// Prefer [`wait_condition!`] for inline predicates so changes to the Rust
8427    /// predicate tokens automatically change the recorded definition
8428    /// fingerprint. Direct callers must provide an equally stable identity in
8429    /// [`ConditionWaitOptions`].
8430    pub fn wait_condition<F>(
8431        &self,
8432        options: ConditionWaitOptions,
8433        predicate: F,
8434    ) -> ConditionWaitCall
8435    where
8436        F: Fn() -> Result<bool> + Send + 'static,
8437    {
8438        ConditionWaitCall {
8439            ctx: self.clone(),
8440            options,
8441            predicate: Box::new(predicate),
8442            occurrence_id: None,
8443            opened_wait: false,
8444            parallel_group_path: Vec::new(),
8445        }
8446    }
8447
8448    /// Wait for server-backed durable time without blocking the worker executor.
8449    ///
8450    /// Polling this future emits one `start_timer` command and yields. The
8451    /// server records the deadline, so neither worker nor server restarts reset
8452    /// the wait. Replay resolves the future only from a `TimerScheduled` and
8453    /// `TimerFired` pair at the same position in the shared durable-command
8454    /// stream, with matching sequence, timer identity, and delay. Sub-second
8455    /// durations round up because protocol deadlines use whole seconds.
8456    ///
8457    /// ```no_run
8458    /// # use durable_workflow::{json, Client, Worker};
8459    /// # use std::time::Duration;
8460    /// # fn configure(client: Client) {
8461    /// let mut worker = Worker::new(client, "rust-workers");
8462    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
8463    ///     ctx.sleep(Duration::from_secs(5)).await?;
8464    ///     Ok(json!({"status": "timer fired"}))
8465    /// });
8466    /// # }
8467    /// ```
8468    pub fn sleep(&self, duration: Duration) -> TimerCall {
8469        let delay_seconds = duration
8470            .as_secs()
8471            .checked_add(u64::from(duration.subsec_nanos() > 0));
8472        TimerCall {
8473            ctx: self.clone(),
8474            delay_seconds,
8475            scheduled: false,
8476            matched_pending: false,
8477            parallel_group_path: Vec::new(),
8478        }
8479    }
8480
8481    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
8482    pub fn start_timer(&self, duration: Duration) -> TimerCall {
8483        self.sleep(duration)
8484    }
8485
8486    /// Evaluate a non-deterministic callback once and durably record its typed value.
8487    ///
8488    /// On replay the callback is not invoked: the value is decoded from the
8489    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
8490    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
8491    /// small values that must remain fixed for the lifetime of a workflow run.
8492    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8493    where
8494        T: Serialize + DeserializeOwned,
8495        F: FnOnce() -> T,
8496    {
8497        {
8498            let mut state = self
8499                .state
8500                .lock()
8501                .map_err(|_| Error::WorkflowStatePoisoned)?;
8502            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8503                return match recorded {
8504                    RecordedCommand::SideEffect { sequence, value } => {
8505                        state.command_cursor += 1;
8506                        value.deserialize().map_err(|error| {
8507                            Error::NonDeterministicReplay(ReplayFailure::new(
8508                                "side_effect_type_mismatch",
8509                                Some(sequence),
8510                                Some(std::any::type_name::<T>().to_string()),
8511                                Some(error.to_string()),
8512                                "recorded side-effect value is incompatible with the requested Rust type",
8513                            ))
8514                        })
8515                    }
8516                    other => Err(command_mismatch(&other, "side effect")),
8517                };
8518            }
8519        }
8520
8521        let value = callback();
8522        let avro_value = AvroValue::from_serialize(&value)?;
8523        let mut state = self
8524            .state
8525            .lock()
8526            .map_err(|_| Error::WorkflowStatePoisoned)?;
8527        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8528        state.commands.push(json!({
8529            "type": "record_side_effect",
8530            "result": result,
8531        }));
8532        Ok(value)
8533    }
8534
8535    /// Record or replay a lossless fixed Avro Value side effect.
8536    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8537    where
8538        F: FnOnce() -> AvroValue,
8539    {
8540        {
8541            let mut state = self
8542                .state
8543                .lock()
8544                .map_err(|_| Error::WorkflowStatePoisoned)?;
8545            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8546                return match recorded {
8547                    RecordedCommand::SideEffect { value, .. } => {
8548                        state.command_cursor += 1;
8549                        Ok(value)
8550                    }
8551                    other => Err(command_mismatch(&other, "side effect")),
8552                };
8553            }
8554        }
8555
8556        let value = callback();
8557        let mut state = self
8558            .state
8559            .lock()
8560            .map_err(|_| Error::WorkflowStatePoisoned)?;
8561        let result = encode_typed_envelope(&value, &state.payload_codec)?;
8562        state.commands.push(json!({
8563            "type": "record_side_effect",
8564            "result": result,
8565        }));
8566        Ok(value)
8567    }
8568
8569    /// Append output items at a deterministic workflow command boundary.
8570    ///
8571    /// Stable idempotency keys are derived from the server-provided durable
8572    /// workflow command identity, command ordinal, and item index. Replay
8573    /// consumes the recorded side effect and never emits another append.
8574    pub fn append_workflow_stream(
8575        &self,
8576        stream_name: impl Into<String>,
8577        items: &[WorkflowStreamAppendItem],
8578        max_pending_items: Option<u64>,
8579    ) -> Result<()> {
8580        if items.is_empty() {
8581            return Err(Error::Codec(
8582                "workflow_stream_items_empty: append requires at least one item".to_string(),
8583            ));
8584        }
8585        if max_pending_items == Some(0) {
8586            return Err(Error::Codec(
8587                "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8588                    .to_string(),
8589            ));
8590        }
8591        let stream_name = stream_name.into();
8592        if stream_name.is_empty() {
8593            return Err(Error::Codec(
8594                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8595            ));
8596        }
8597
8598        let mut state = self
8599            .state
8600            .lock()
8601            .map_err(|_| Error::WorkflowStatePoisoned)?;
8602        let command_ordinal = state.workflow_stream_command_counter;
8603        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8604            state.workflow_stream_command_counter += 1;
8605            return match recorded {
8606                RecordedCommand::SideEffect { .. } => {
8607                    state.command_cursor += 1;
8608                    Ok(())
8609                }
8610                other => Err(command_mismatch(&other, "workflow stream append")),
8611            };
8612        }
8613
8614        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8615        state.workflow_stream_command_counter += 1;
8616        let wire_items = items
8617            .iter()
8618            .enumerate()
8619            .map(|(item_index, item)| {
8620                item.wire_value(Some(format!(
8621                    "dw-stream:{identity}:{command_ordinal}:{item_index}"
8622                )))
8623            })
8624            .collect::<Vec<_>>();
8625        let mut directive = json!({
8626            "operation": "append",
8627            "stream_name": stream_name,
8628            "command_identity": identity,
8629            "command_ordinal": command_ordinal,
8630            "items": wire_items,
8631        });
8632        if let Some(max_pending_items) = max_pending_items {
8633            directive["max_pending_items"] = json!(max_pending_items);
8634        }
8635        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8636        state.commands.push(json!({
8637            "type": "record_side_effect",
8638            "result": result,
8639            "workflow_stream": directive,
8640        }));
8641        Ok(())
8642    }
8643
8644    /// Close a run-scoped output stream at a deterministic command boundary.
8645    pub fn close_workflow_stream(
8646        &self,
8647        stream_name: impl Into<String>,
8648        retention_seconds: Option<u64>,
8649    ) -> Result<()> {
8650        self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8651    }
8652
8653    /// Mark a run-scoped output stream errored at a deterministic command boundary.
8654    pub fn error_workflow_stream(
8655        &self,
8656        stream_name: impl Into<String>,
8657        error_reason: impl Into<String>,
8658        retention_seconds: Option<u64>,
8659    ) -> Result<()> {
8660        let error_reason = error_reason.into();
8661        if error_reason.is_empty() {
8662            return Err(Error::Codec(
8663                "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8664            ));
8665        }
8666        self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8667    }
8668
8669    fn finish_workflow_stream(
8670        &self,
8671        stream_name: String,
8672        error_reason: Option<String>,
8673        retention_seconds: Option<u64>,
8674    ) -> Result<()> {
8675        if stream_name.is_empty() {
8676            return Err(Error::Codec(
8677                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8678            ));
8679        }
8680        if retention_seconds == Some(0) {
8681            return Err(Error::Codec(
8682                "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8683            ));
8684        }
8685        let mut state = self
8686            .state
8687            .lock()
8688            .map_err(|_| Error::WorkflowStatePoisoned)?;
8689        let command_ordinal = state.workflow_stream_command_counter;
8690        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8691            state.workflow_stream_command_counter += 1;
8692            return match recorded {
8693                RecordedCommand::SideEffect { .. } => {
8694                    state.command_cursor += 1;
8695                    Ok(())
8696                }
8697                other => Err(command_mismatch(&other, "workflow stream close")),
8698            };
8699        }
8700        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8701        state.workflow_stream_command_counter += 1;
8702        let mut directive = json!({
8703            "operation": if error_reason.is_some() { "error" } else { "close" },
8704            "stream_name": stream_name,
8705            "command_identity": identity,
8706            "command_ordinal": command_ordinal,
8707        });
8708        if let Some(error_reason) = error_reason {
8709            directive["error_reason"] = json!(error_reason);
8710        }
8711        if let Some(retention_seconds) = retention_seconds {
8712            directive["retention_seconds"] = json!(retention_seconds);
8713        }
8714        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8715        state.commands.push(json!({
8716            "type": "record_side_effect",
8717            "result": result,
8718            "workflow_stream": directive,
8719        }));
8720        Ok(())
8721    }
8722
8723    fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8724        let identity = state.workflow_command_identity.as_str();
8725        if identity.is_empty() {
8726            return Err(Error::MissingWorkflowCommandIdentity);
8727        }
8728        Ok(identity)
8729    }
8730
8731    /// Validate, emit, or replay a typed workflow search-attribute update.
8732    ///
8733    /// The command is non-blocking within a workflow decision, but its
8734    /// `SearchAttributesUpserted` event occupies the same deterministic command
8735    /// stream as activities, timers, conditions, and other durable operations.
8736    pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8737        update.validate()?;
8738        let (attributes, attribute_types) = update.into_wire_parts();
8739        let mut state = self
8740            .state
8741            .lock()
8742            .map_err(|_| Error::WorkflowStatePoisoned)?;
8743
8744        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8745            return match recorded {
8746                RecordedCommand::SearchAttributes {
8747                    sequence,
8748                    attributes: recorded_attributes,
8749                    attribute_types: recorded_attribute_types,
8750                } => {
8751                    if recorded_attributes != attributes {
8752                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8753                            "search_attribute_value_mismatch",
8754                            Some(sequence),
8755                            Some(recorded_attributes.to_string()),
8756                            Some(attributes.to_string()),
8757                            "search-attribute values differ from the recorded durable command",
8758                        )));
8759                    }
8760                    if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8761                        if recorded_types != attribute_types {
8762                            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8763                                "search_attribute_type_mismatch",
8764                                Some(sequence),
8765                                Some(json!(recorded_types).to_string()),
8766                                Some(json!(attribute_types).to_string()),
8767                                "search-attribute declared types differ from the recorded durable command",
8768                            )));
8769                        }
8770                    }
8771                    state.command_cursor += 1;
8772                    Ok(())
8773                }
8774                other => Err(command_mismatch(&other, "search-attribute update")),
8775            };
8776        }
8777
8778        let mut command = serde_json::Map::from_iter([
8779            ("type".to_string(), json!("upsert_search_attributes")),
8780            ("attributes".to_string(), attributes),
8781        ]);
8782        if !attribute_types.is_empty() {
8783            command.insert("attribute_types".to_string(), json!(attribute_types));
8784        }
8785        state.commands.push(Value::Object(command));
8786        Ok(())
8787    }
8788
8789    /// Record a UUIDv4 once and return the same UUID on every replay.
8790    pub fn uuid_v4(&self) -> Result<Uuid> {
8791        self.side_effect(Uuid::new_v4)
8792    }
8793
8794    /// Select the newest supported version for a change, or replay the version
8795    /// already committed for that stable change ID.
8796    pub fn get_version(
8797        &self,
8798        change_id: impl Into<String>,
8799        min_supported: i32,
8800        max_supported: i32,
8801    ) -> Result<i32> {
8802        let change_id = change_id.into();
8803        if change_id.trim().is_empty() {
8804            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8805                "version_change_id_invalid",
8806                None,
8807                Some("non-empty change ID".to_string()),
8808                Some(change_id),
8809                "version markers require a stable non-empty change ID",
8810            )));
8811        }
8812        if min_supported > max_supported {
8813            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8814                "version_range_invalid",
8815                None,
8816                Some("min_supported <= max_supported".to_string()),
8817                Some(format!("{min_supported}..={max_supported}")),
8818                "version marker supported range is invalid",
8819            )));
8820        }
8821
8822        let mut state = self
8823            .state
8824            .lock()
8825            .map_err(|_| Error::WorkflowStatePoisoned)?;
8826        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8827            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8828            return Ok(version);
8829        }
8830
8831        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8832            return match recorded {
8833                RecordedCommand::VersionMarker {
8834                    sequence,
8835                    change_id: recorded_change_id,
8836                    version,
8837                    ..
8838                } => {
8839                    if recorded_change_id != change_id {
8840                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8841                            "version_change_id_mismatch",
8842                            Some(sequence),
8843                            Some(recorded_change_id),
8844                            Some(change_id),
8845                            "recorded version marker change ID differs from current workflow code",
8846                        )));
8847                    }
8848                    ensure_version_supported(
8849                        &change_id,
8850                        version,
8851                        min_supported,
8852                        max_supported,
8853                        sequence,
8854                    )?;
8855                    state.command_cursor += 1;
8856                    state.version_markers.insert(change_id, (version, sequence));
8857                    Ok(version)
8858                }
8859                other => Err(command_mismatch(
8860                    &other,
8861                    format!("version marker:{change_id}"),
8862                )),
8863            };
8864        }
8865
8866        let version = max_supported;
8867        state.commands.push(json!({
8868            "type": "record_version_marker",
8869            "change_id": change_id,
8870            "version": version,
8871            "min_supported": min_supported,
8872            "max_supported": max_supported,
8873        }));
8874        // Sequence numbers are assigned by the server. Zero identifies a marker
8875        // selected in this uncommitted decision batch for duplicate-call checks.
8876        state.version_markers.insert(change_id, (version, 0));
8877        Ok(version)
8878    }
8879
8880    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
8881    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8882        Ok(self.get_version(change_id, -1, 1)? == 1)
8883    }
8884
8885    /// Keep a patch marker in history after the legacy branch has been removed.
8886    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8887        self.get_version(change_id, -1, 1).map(|_| ())
8888    }
8889
8890    /// Merge non-indexed workflow memo metadata through durable history.
8891    ///
8892    /// Avro `null` deletes a key. The SDK encodes the complete patch in the
8893    /// public Avro payload envelope consumed by Server and Cloud runtimes.
8894    pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8895        let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8896        let mut state = self
8897            .state
8898            .lock()
8899            .map_err(|_| Error::WorkflowStatePoisoned)?;
8900
8901        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8902            return match recorded {
8903                RecordedCommand::Memo {
8904                    sequence,
8905                    entries: recorded_entries,
8906                } => {
8907                    if recorded_entries != entries {
8908                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8909                            "memo_update_mismatch",
8910                            Some(sequence),
8911                            Some(format!("{recorded_entries:?}")),
8912                            Some(format!("{entries:?}")),
8913                            "recorded memo entries differ from the current workflow update",
8914                        )));
8915                    }
8916                    state.command_cursor += 1;
8917                    Ok(())
8918                }
8919                other => Err(command_mismatch(&other, "memo upsert")),
8920            };
8921        }
8922
8923        let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8924        state.commands.push(json!({
8925            "type": "upsert_memo",
8926            "entries": entries_envelope,
8927        }));
8928        Ok(())
8929    }
8930
8931    /// Start a named durable child on an explicit queue and await its result.
8932    ///
8933    /// The command is recorded in the parent's sequence-ordered durable command
8934    /// stream. Replay keeps a scheduled child pending without emitting another
8935    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
8936    /// values preserve the history payload codec and include both sides of the
8937    /// durable relationship; failures are returned as
8938    /// [`Error::ChildWorkflowFailed`].
8939    ///
8940    /// ```no_run
8941    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
8942    /// # fn configure(client: Client) {
8943    /// let mut worker = Worker::new(client, "parent-workers");
8944    /// worker.register_workflow("order-parent", |ctx, _input| async move {
8945    ///     let child = ctx
8946    ///         .start_child_workflow(
8947    ///             "fulfil-order",
8948    ///             ChildWorkflowOptions::new("fulfilment-workers")
8949    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
8950    ///             json!([{"order_id": "order-42"}]),
8951    ///         )
8952    ///         .await?;
8953    ///     Ok(child.result)
8954    /// });
8955    /// # }
8956    /// ```
8957    pub fn start_child_workflow<T: Serialize>(
8958        &self,
8959        workflow_type: impl Into<String>,
8960        options: ChildWorkflowOptions,
8961        args: T,
8962    ) -> ChildWorkflowCall {
8963        ChildWorkflowCall {
8964            ctx: self.clone(),
8965            workflow_type: workflow_type.into(),
8966            options,
8967            args: Some(AvroValue::from_serialize(&args)),
8968            scheduled: false,
8969            matched_pending: false,
8970            parallel_group_path: Vec::new(),
8971        }
8972    }
8973
8974    pub async fn start_child_workflow_avro_value<T: Serialize>(
8975        &self,
8976        workflow_type: impl Into<String>,
8977        options: ChildWorkflowOptions,
8978        args: T,
8979    ) -> Result<ChildWorkflowAvroResult> {
8980        let mut call = self.start_child_workflow(workflow_type, options, args);
8981        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8982    }
8983
8984    fn take_commands(&self) -> Result<Vec<Value>> {
8985        let mut state = self
8986            .state
8987            .lock()
8988            .map_err(|_| Error::WorkflowStatePoisoned)?;
8989        Ok(std::mem::take(&mut state.commands))
8990    }
8991
8992    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8993        let mut state = self
8994            .state
8995            .lock()
8996            .map_err(|_| Error::WorkflowStatePoisoned)?;
8997
8998        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8999            return Err(command_mismatch(&recorded, "continue as new"));
9000        }
9001        if state.recorded_continue_as_new_sequence.is_some() {
9002            state.continue_as_new_consumed = true;
9003            return Ok(None);
9004        }
9005
9006        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
9007        let mut command = serde_json::Map::from_iter([
9008            ("type".to_string(), json!("continue_as_new")),
9009            ("arguments".to_string(), arguments),
9010            ("queue".to_string(), json!(state.task_queue.clone())),
9011        ]);
9012        if let Some(workflow_type) = request.options.workflow_type {
9013            command.insert("workflow_type".to_string(), json!(workflow_type));
9014        }
9015        if let Some(task_queue) = request.options.task_queue {
9016            command.insert("queue".to_string(), json!(task_queue));
9017        }
9018        Ok(Some(Value::Object(command)))
9019    }
9020
9021    fn matched_recorded_pending(&self) -> Result<bool> {
9022        let state = self
9023            .state
9024            .lock()
9025            .map_err(|_| Error::WorkflowStatePoisoned)?;
9026        Ok(state.matched_recorded_pending)
9027    }
9028
9029    fn ensure_history_consumed(&self) -> Result<()> {
9030        let state = self
9031            .state
9032            .lock()
9033            .map_err(|_| Error::WorkflowStatePoisoned)?;
9034        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
9035            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9036                "recorded_commands_unconsumed",
9037                Some(command.sequence()),
9038                Some(command.shape().to_string()),
9039                Some("workflow completion".to_string()),
9040                "workflow completed before consuming all recorded durable commands",
9041            )));
9042        }
9043        if let Some(sequence) = state
9044            .recorded_continue_as_new_sequence
9045            .filter(|_| !state.continue_as_new_consumed)
9046        {
9047            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9048                "recorded_continue_as_new_unconsumed",
9049                Some(sequence),
9050                Some("continue as new".to_string()),
9051                Some("workflow completion".to_string()),
9052                "workflow completed without consuming its recorded continue-as-new transition",
9053            )));
9054        }
9055        Ok(())
9056    }
9057}
9058
9059fn contiguous_message_stream_count(
9060    pending: &[MessageStreamMessage],
9061    cursor: u64,
9062    max_items: usize,
9063) -> usize {
9064    pending
9065        .iter()
9066        .take(max_items)
9067        .enumerate()
9068        .take_while(|(offset, message)| {
9069            u64::try_from(*offset)
9070                .ok()
9071                .and_then(|offset| cursor.checked_add(offset + 1))
9072                == Some(message.position)
9073        })
9074        .count()
9075}
9076
9077fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
9078    matches!(
9079        event.event_type.as_str(),
9080        "ActivityScheduled"
9081            | "TimerScheduled"
9082            | "ChildWorkflowScheduled"
9083            | "SignalWaitOpened"
9084            | "ConditionWaitOpened"
9085            | "SearchAttributesUpserted"
9086            | "SideEffectRecorded"
9087            | "VersionMarkerRecorded"
9088            | "MemoUpserted"
9089            | "WorkflowContinuedAsNew"
9090    )
9091}
9092
9093#[derive(Debug)]
9094struct WorkflowState {
9095    workflow_id: Option<String>,
9096    run_id: Option<String>,
9097    task_queue: String,
9098    payload_codec: String,
9099    history_events: Arc<Vec<HistoryEvent>>,
9100    history_budget: WorkflowHistoryBudget,
9101    cancel_requested: bool,
9102    resume_signal: Option<ResumeSignal>,
9103    recorded_commands: Vec<RecordedCommand>,
9104    selection_markers: Vec<SelectionMarker>,
9105    selection_marker_cursor: usize,
9106    cancelled_selection_members: Vec<SelectionCancellation>,
9107    recorded_continue_as_new_sequence: Option<u64>,
9108    continue_as_new_consumed: bool,
9109    command_cursor: usize,
9110    condition_wait_occurrence_counter: u64,
9111    matched_recorded_pending: bool,
9112    version_markers: HashMap<String, (i32, u64)>,
9113    workflow_command_identity: String,
9114    workflow_stream_command_counter: u64,
9115    commands: Vec<Value>,
9116    message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9117    message_stream_cursors: HashMap<String, u64>,
9118    message_stream_waits: HashMap<String, u64>,
9119}
9120
9121impl WorkflowState {
9122    #[cfg(test)]
9123    fn new(
9124        history: Vec<HistoryEvent>,
9125        task_queue: String,
9126        payload_codec: String,
9127        resume_signal: Option<ResumeSignal>,
9128    ) -> Result<Self> {
9129        Self::new_with_identity(
9130            history,
9131            None,
9132            None,
9133            task_queue,
9134            payload_codec,
9135            resume_signal,
9136        )
9137    }
9138
9139    fn new_with_identity(
9140        history: Vec<HistoryEvent>,
9141        workflow_id: Option<String>,
9142        run_id: Option<String>,
9143        task_queue: String,
9144        payload_codec: String,
9145        resume_signal: Option<ResumeSignal>,
9146    ) -> Result<Self> {
9147        let recorded_commands = recorded_commands(
9148            &history,
9149            &payload_codec,
9150            WorkflowIdentity {
9151                workflow_id: workflow_id.clone(),
9152                run_id: run_id.clone(),
9153            },
9154        )?;
9155        let selection_markers = recorded_selection_markers(&history)?;
9156        let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9157        let recorded_continue_as_new = history
9158            .iter()
9159            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9160            .collect::<Vec<_>>();
9161        if recorded_continue_as_new.len() > 1 {
9162            return Err(invalid_recorded_history(
9163                "duplicate_continue_as_new_transition",
9164                recorded_continue_as_new
9165                    .last()
9166                    .and_then(|event| durable_event_sequence(event))
9167                    .unwrap_or(0),
9168                "one WorkflowContinuedAsNew event",
9169                &format!(
9170                    "{} WorkflowContinuedAsNew events",
9171                    recorded_continue_as_new.len()
9172                ),
9173                "workflow history records one continue-as-new transition more than once",
9174            ));
9175        }
9176        let recorded_continue_as_new_sequence = recorded_continue_as_new
9177            .first()
9178            .map(|event| {
9179                durable_event_sequence(event).ok_or_else(|| {
9180                    Error::NonDeterministicReplay(ReplayFailure::new(
9181                        "continue_as_new_sequence_missing",
9182                        None,
9183                        Some("recorded transition sequence".to_string()),
9184                        Some("missing sequence".to_string()),
9185                        "WorkflowContinuedAsNew history is missing its recorded sequence",
9186                    ))
9187                })
9188            })
9189            .transpose()?;
9190        let mut message_stream_cursors = HashMap::new();
9191        for event in &history {
9192            if !matches!(
9193                event.event_type.as_str(),
9194                "SignalReceived" | "SignalApplied"
9195            ) || event.payload.get("signal_name").and_then(Value::as_str)
9196                != Some(MESSAGE_STREAM_SIGNAL)
9197            {
9198                continue;
9199            }
9200            let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9201            if arguments.len() != 1 {
9202                continue;
9203            }
9204            let envelope = arguments[0].clone().into_json()?;
9205            let Some(envelope) = envelope.as_object() else {
9206                continue;
9207            };
9208            if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9209            {
9210                continue;
9211            }
9212            let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9213                continue;
9214            };
9215            let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9216            else {
9217                continue;
9218            };
9219            let cursor = message_stream_cursors
9220                .entry(stream_name.to_string())
9221                .or_insert(0);
9222            *cursor = (*cursor).max(through_position);
9223        }
9224        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9225        let cancel_requested = history.iter().any(|event| {
9226            matches!(
9227                event.event_type.as_str(),
9228                "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9229            )
9230        });
9231        Ok(Self {
9232            workflow_command_identity: String::new(),
9233            workflow_stream_command_counter: 0,
9234            workflow_id,
9235            run_id,
9236            task_queue,
9237            payload_codec,
9238            history_events: Arc::new(history),
9239            history_budget: WorkflowHistoryBudget {
9240                event_count,
9241                ..WorkflowHistoryBudget::default()
9242            },
9243            cancel_requested,
9244            resume_signal,
9245            recorded_commands,
9246            selection_markers,
9247            selection_marker_cursor: 0,
9248            cancelled_selection_members,
9249            recorded_continue_as_new_sequence,
9250            continue_as_new_consumed: false,
9251            command_cursor: 0,
9252            condition_wait_occurrence_counter: 0,
9253            matched_recorded_pending: false,
9254            version_markers: HashMap::new(),
9255            commands: Vec::new(),
9256            message_stream_messages: HashMap::new(),
9257            message_stream_cursors,
9258            message_stream_waits: HashMap::new(),
9259        })
9260    }
9261}
9262
9263enum MessageStreamDelivery {
9264    Message(MessageStreamMessage),
9265    Cursor {
9266        stream_name: String,
9267        through_position: u64,
9268    },
9269}
9270
9271fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9272    if arguments.len() != 1 {
9273        return Ok(None);
9274    }
9275    let envelope = arguments
9276        .into_iter()
9277        .next()
9278        .expect("one argument was checked");
9279    let Some(envelope) = envelope.as_object() else {
9280        return Ok(None);
9281    };
9282    let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9283        return Ok(None);
9284    };
9285    if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9286        let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9287        else {
9288            return Ok(None);
9289        };
9290        return Ok(Some(MessageStreamDelivery::Cursor {
9291            stream_name: stream_name.to_string(),
9292            through_position,
9293        }));
9294    }
9295    if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9296        return Ok(None);
9297    }
9298    let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9299        return Ok(None);
9300    };
9301    let Some(position) = envelope
9302        .get("position")
9303        .and_then(Value::as_u64)
9304        .filter(|value| *value > 0)
9305    else {
9306        return Ok(None);
9307    };
9308    let Some(payload_envelope) = envelope.get("payload_envelope") else {
9309        return Ok(None);
9310    };
9311    let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9312    else {
9313        return Ok(None);
9314    };
9315    let decoded = decode_avro_value(&payload_envelope)?;
9316    let AvroValue::Array(values) = decoded else {
9317        return Ok(None);
9318    };
9319    Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9320        stream_name: stream_name.to_string(),
9321        message_id: message_id.to_string(),
9322        position,
9323        arguments: values,
9324    })))
9325}
9326
9327#[derive(Clone, Debug)]
9328enum RecordedCommand {
9329    Activity {
9330        sequence: u64,
9331        activity_type: Option<String>,
9332        options: Option<RecordedActivityOptions>,
9333        outcome: Option<ActivityOutcome>,
9334        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9335    },
9336    Timer {
9337        sequence: u64,
9338        delay_seconds: u64,
9339        fired: bool,
9340        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9341    },
9342    ChildWorkflow {
9343        sequence: u64,
9344        workflow_type: Option<String>,
9345        outcome: Option<ChildWorkflowOutcome>,
9346        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9347    },
9348    SignalWait {
9349        sequence: u64,
9350        signal_name: String,
9351        value: Option<Vec<AvroValue>>,
9352        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9353    },
9354    ConditionWait {
9355        sequence: u64,
9356        occurrence_id: String,
9357        condition_key: Option<String>,
9358        predicate_identity: String,
9359        timeout_seconds: Option<u64>,
9360        result: Option<ConditionWaitResult>,
9361        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9362    },
9363    SearchAttributes {
9364        sequence: u64,
9365        attributes: Value,
9366        attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9367    },
9368    SideEffect {
9369        sequence: u64,
9370        value: AvroValue,
9371    },
9372    VersionMarker {
9373        sequence: u64,
9374        change_id: String,
9375        version: i32,
9376    },
9377    Memo {
9378        sequence: u64,
9379        entries: AvroValue,
9380    },
9381}
9382
9383#[derive(Clone, Debug, PartialEq, Eq)]
9384struct SelectionMarker {
9385    selection_group_id: String,
9386    selection_group_base_sequence: u64,
9387    selection_group_size: usize,
9388    member_key: SelectionKey,
9389    member_index: usize,
9390    member_base_sequence: u64,
9391    member_size: usize,
9392    operation_kind: String,
9393    operation_identity: String,
9394    outcome: String,
9395    resolution_event_id: String,
9396    resolution_event_type: String,
9397}
9398
9399#[derive(Clone, Debug, PartialEq, Eq)]
9400struct SelectionCancellation {
9401    selection_group_id: String,
9402    member_key: SelectionKey,
9403    member_index: usize,
9404    member_base_sequence: u64,
9405    member_size: usize,
9406    operation_kind: String,
9407    operation_identity: String,
9408}
9409
9410fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9411    let mut markers: Vec<SelectionMarker> = Vec::new();
9412    for event in events
9413        .iter()
9414        .filter(|event| event.event_type == "SelectionResolved")
9415    {
9416        let payload = &event.payload;
9417        let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9418        let group_size = required_selection_usize(payload, "selection_group_size")?;
9419        let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9420        let member_size = required_selection_usize(payload, "member_size")?;
9421        let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9422        let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9423            invalid_recorded_history(
9424                "selection_marker_invalid",
9425                base_sequence,
9426                "non-empty selection_group_id",
9427                &payload.to_string(),
9428                "selection winner history is missing its durable group identity",
9429            )
9430        })?;
9431        let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9432        if group_id != expected_group_id {
9433            return Err(invalid_recorded_history(
9434                "selection_marker_invalid",
9435                base_sequence,
9436                &expected_group_id,
9437                &group_id,
9438                "selection winner history contains an incompatible group identity",
9439            ));
9440        }
9441        let group_end = base_sequence
9442            .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9443            .unwrap_or(u64::MAX);
9444        let member_end = member_base_sequence
9445            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9446            .unwrap_or(u64::MAX);
9447        if member_index >= group_size
9448            || member_base_sequence < base_sequence
9449            || member_end > group_end
9450        {
9451            return Err(invalid_recorded_history(
9452                "selection_marker_invalid",
9453                base_sequence,
9454                "winner member within selection group bounds",
9455                &payload.to_string(),
9456                "selection winner history contains an invalid member range",
9457            ));
9458        }
9459        let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9460            invalid_recorded_history(
9461                "selection_marker_invalid",
9462                base_sequence,
9463                "selection operation kind",
9464                &payload.to_string(),
9465                "selection winner history is missing its operation kind",
9466            )
9467        })?;
9468        if !matches!(
9469            operation_kind.as_str(),
9470            "activity" | "child" | "timer" | "signal" | "condition" | "group"
9471        ) {
9472            return Err(invalid_recorded_history(
9473                "selection_marker_invalid",
9474                base_sequence,
9475                "activity, child, timer, signal, condition, or group",
9476                &operation_kind,
9477                "selection winner history contains an unsupported operation kind",
9478            ));
9479        }
9480        let operation_identity =
9481            payload_string(payload, "operation_identity").ok_or_else(|| {
9482                invalid_recorded_history(
9483                    "selection_marker_invalid",
9484                    base_sequence,
9485                    "non-empty operation identity",
9486                    &payload.to_string(),
9487                    "selection winner history is missing its durable operation identity",
9488                )
9489            })?;
9490        let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9491            invalid_recorded_history(
9492                "selection_marker_invalid",
9493                base_sequence,
9494                "completed or failed selection outcome",
9495                &payload.to_string(),
9496                "selection winner history is missing its outcome",
9497            )
9498        })?;
9499        if !matches!(outcome.as_str(), "completed" | "failed") {
9500            return Err(invalid_recorded_history(
9501                "selection_marker_invalid",
9502                base_sequence,
9503                "completed or failed selection outcome",
9504                &outcome,
9505                "selection winner history contains an unsupported outcome",
9506            ));
9507        }
9508        let marker = SelectionMarker {
9509            selection_group_id: group_id,
9510            selection_group_base_sequence: base_sequence,
9511            selection_group_size: group_size,
9512            member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9513            member_index,
9514            member_base_sequence,
9515            member_size,
9516            operation_kind,
9517            operation_identity,
9518            outcome,
9519            resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9520                || {
9521                    invalid_recorded_history(
9522                        "selection_marker_invalid",
9523                        base_sequence,
9524                        "durable resolution_event_id",
9525                        &payload.to_string(),
9526                        "selection winner history is missing its terminal event identity",
9527                    )
9528                },
9529            )?,
9530            resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9531                || {
9532                    invalid_recorded_history(
9533                        "selection_marker_invalid",
9534                        base_sequence,
9535                        "durable resolution_event_type",
9536                        &payload.to_string(),
9537                        "selection winner history is missing its terminal event type",
9538                    )
9539                },
9540            )?,
9541        };
9542        if let Some(existing) = markers
9543            .iter()
9544            .find(|existing| existing.selection_group_id == marker.selection_group_id)
9545        {
9546            if existing != &marker {
9547                return Err(invalid_recorded_history(
9548                    "selection_marker_conflict",
9549                    base_sequence,
9550                    &format!("one winner for {}", marker.selection_group_id),
9551                    &payload.to_string(),
9552                    "selection history records conflicting winners for one durable group",
9553                ));
9554            }
9555            continue;
9556        }
9557        markers.push(marker);
9558    }
9559    Ok(markers)
9560}
9561
9562fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9563    let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9564    for event in events
9565        .iter()
9566        .filter(|event| event.event_type == "SelectionOperationCancelled")
9567    {
9568        let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9569            invalid_recorded_history(
9570                "selection_cancellation_invalid",
9571                0,
9572                "non-empty selection_group_id",
9573                &event.payload.to_string(),
9574                "selection cancellation history is missing its group identity",
9575            )
9576        })?;
9577        let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9578        let marker = SelectionCancellation {
9579            selection_group_id: group_id,
9580            member_key: selection_key_from_value(
9581                event.payload.get("member_key"),
9582                member_base_sequence,
9583            )?,
9584            member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9585            member_base_sequence,
9586            member_size: required_selection_usize(&event.payload, "member_size")?,
9587            operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9588                invalid_recorded_history(
9589                    "selection_cancellation_invalid",
9590                    member_base_sequence,
9591                    "selection operation kind",
9592                    &event.payload.to_string(),
9593                    "selection cancellation is missing its operation kind",
9594                )
9595            })?,
9596            operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9597                || {
9598                    invalid_recorded_history(
9599                        "selection_cancellation_invalid",
9600                        member_base_sequence,
9601                        "selection operation identity",
9602                        &event.payload.to_string(),
9603                        "selection cancellation is missing its operation identity",
9604                    )
9605                },
9606            )?,
9607        };
9608        if let Some(existing) = cancelled.iter().find(|recorded| {
9609            recorded.selection_group_id == marker.selection_group_id
9610                && recorded.member_base_sequence == marker.member_base_sequence
9611        }) {
9612            if existing != &marker {
9613                return Err(invalid_recorded_history(
9614                    "selection_cancellation_conflict",
9615                    member_base_sequence,
9616                    "one stable SelectionOperationCancelled marker",
9617                    &event.payload.to_string(),
9618                    "selection cancellation history contains conflicting member metadata",
9619                ));
9620            }
9621        } else {
9622            cancelled.push(marker);
9623        }
9624    }
9625    Ok(cancelled)
9626}
9627
9628fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9629    payload
9630        .get(field)
9631        .and_then(value_as_u64)
9632        .filter(|value| *value > 0)
9633        .ok_or_else(|| {
9634            invalid_recorded_history(
9635                "selection_marker_invalid",
9636                0,
9637                &format!("positive integer {field}"),
9638                &payload.to_string(),
9639                "selection history contains invalid durable identity metadata",
9640            )
9641        })
9642}
9643
9644fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9645    required_selection_usize_allow_zero(payload, field).and_then(|value| {
9646        if value > 0 {
9647            Ok(value)
9648        } else {
9649            Err(invalid_recorded_history(
9650                "selection_marker_invalid",
9651                0,
9652                &format!("positive integer {field}"),
9653                &payload.to_string(),
9654                "selection history contains invalid durable identity metadata",
9655            ))
9656        }
9657    })
9658}
9659
9660fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9661    payload
9662        .get(field)
9663        .and_then(value_as_u64)
9664        .and_then(|value| usize::try_from(value).ok())
9665        .ok_or_else(|| {
9666            invalid_recorded_history(
9667                "selection_marker_invalid",
9668                0,
9669                &format!("non-negative integer {field}"),
9670                &payload.to_string(),
9671                "selection history contains invalid durable identity metadata",
9672            )
9673        })
9674}
9675
9676#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9677struct RecordedActivityOptions {
9678    task_queue: RecordedSnapshotValue<Option<String>>,
9679    execution_mode: RecordedSnapshotValue<Option<String>>,
9680    retry_policy: ActivityRetrySnapshot,
9681}
9682
9683#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9684enum RecordedSnapshotValue<T> {
9685    /// Older history did not persist this field, so it cannot constrain replay.
9686    Unknown,
9687    Known(T),
9688}
9689
9690impl<T: PartialEq> RecordedSnapshotValue<T> {
9691    fn matches_current(&self, current: &Self) -> bool {
9692        match self {
9693            Self::Unknown => true,
9694            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9695        }
9696    }
9697}
9698
9699#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9700struct ActivityRetrySnapshot {
9701    snapshot_version: RecordedSnapshotValue<Option<u64>>,
9702    max_attempts: RecordedSnapshotValue<Option<u64>>,
9703    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9704    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9705    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9706    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9707    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9708    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9709}
9710
9711impl ActivityRetrySnapshot {
9712    fn matches_current(&self, current: &Self) -> bool {
9713        self.snapshot_version
9714            .matches_current(&current.snapshot_version)
9715            && self.max_attempts.matches_current(&current.max_attempts)
9716            && self
9717                .backoff_seconds
9718                .matches_current(&current.backoff_seconds)
9719            && self
9720                .start_to_close_timeout
9721                .matches_current(&current.start_to_close_timeout)
9722            && self
9723                .schedule_to_start_timeout
9724                .matches_current(&current.schedule_to_start_timeout)
9725            && self
9726                .schedule_to_close_timeout
9727                .matches_current(&current.schedule_to_close_timeout)
9728            && self
9729                .heartbeat_timeout
9730                .matches_current(&current.heartbeat_timeout)
9731            && self
9732                .non_retryable_error_types
9733                .matches_current(&current.non_retryable_error_types)
9734    }
9735}
9736
9737fn recorded_optional_u64(
9738    object: Option<&serde_json::Map<String, Value>>,
9739    field: &str,
9740) -> RecordedSnapshotValue<Option<u64>> {
9741    match object.and_then(|object| object.get(field)) {
9742        None => RecordedSnapshotValue::Unknown,
9743        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9744        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9745    }
9746}
9747
9748fn recorded_optional_string(
9749    object: &serde_json::Map<String, Value>,
9750    field: &str,
9751) -> RecordedSnapshotValue<Option<String>> {
9752    match object.get(field) {
9753        None => RecordedSnapshotValue::Unknown,
9754        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9755        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9756    }
9757}
9758
9759fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9760    let policy = policy.and_then(Value::as_object);
9761    let backoff_seconds = policy
9762        .and_then(|policy| policy.get("backoff_seconds"))
9763        .and_then(Value::as_array)
9764        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9765        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9766    let mut non_retryable_error_types = Vec::new();
9767    for error_type in policy
9768        .and_then(|policy| policy.get("non_retryable_error_types"))
9769        .and_then(Value::as_array)
9770        .into_iter()
9771        .flatten()
9772        .filter_map(Value::as_str)
9773        .map(str::trim)
9774        .filter(|error_type| !error_type.is_empty())
9775    {
9776        if !non_retryable_error_types
9777            .iter()
9778            .any(|recorded| recorded == error_type)
9779        {
9780            non_retryable_error_types.push(error_type.to_string());
9781        }
9782    }
9783
9784    ActivityRetrySnapshot {
9785        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9786        max_attempts: recorded_optional_u64(policy, "max_attempts"),
9787        backoff_seconds,
9788        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9789        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9790        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9791        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9792        non_retryable_error_types: if policy
9793            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9794        {
9795            RecordedSnapshotValue::Known(non_retryable_error_types)
9796        } else {
9797            RecordedSnapshotValue::Unknown
9798        },
9799    }
9800}
9801
9802fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9803    let policy = options.retry_policy.as_ref();
9804    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9805        Some(Value::Null) => None,
9806        Some(value) => value_as_u64(value),
9807        None => Some(1),
9808    };
9809    let backoff_seconds = policy
9810        .and_then(|policy| policy.get("backoff_seconds"))
9811        .and_then(Value::as_array)
9812        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9813        .unwrap_or_default();
9814    let non_retryable_error_types = policy
9815        .and_then(|policy| policy.get("non_retryable_error_types"))
9816        .and_then(Value::as_array)
9817        .into_iter()
9818        .flatten()
9819        .filter_map(Value::as_str)
9820        .map(str::to_string)
9821        .collect();
9822
9823    ActivityRetrySnapshot {
9824        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9825        max_attempts: RecordedSnapshotValue::Known(max_attempts),
9826        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9827        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9828        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9829        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9830        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9831        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9832    }
9833}
9834
9835fn activity_options_description(options: &RecordedActivityOptions) -> String {
9836    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9837}
9838
9839impl RecordedCommand {
9840    fn sequence(&self) -> u64 {
9841        match self {
9842            Self::Activity { sequence, .. }
9843            | Self::Timer { sequence, .. }
9844            | Self::ChildWorkflow { sequence, .. }
9845            | Self::SignalWait { sequence, .. }
9846            | Self::ConditionWait { sequence, .. }
9847            | Self::SearchAttributes { sequence, .. }
9848            | Self::SideEffect { sequence, .. }
9849            | Self::VersionMarker { sequence, .. }
9850            | Self::Memo { sequence, .. } => *sequence,
9851        }
9852    }
9853
9854    fn shape(&self) -> &'static str {
9855        match self {
9856            Self::Activity { .. } => "activity",
9857            Self::Timer { .. } => "timer",
9858            Self::ChildWorkflow { .. } => "child workflow",
9859            Self::SignalWait { .. } => "signal wait",
9860            Self::ConditionWait { .. } => "condition wait",
9861            Self::SearchAttributes { .. } => "search-attribute update",
9862            Self::SideEffect { .. } => "side effect",
9863            Self::VersionMarker { .. } => "version marker",
9864            Self::Memo { .. } => "memo upsert",
9865        }
9866    }
9867}
9868
9869fn ensure_version_supported(
9870    change_id: &str,
9871    version: i32,
9872    min_supported: i32,
9873    max_supported: i32,
9874    sequence: u64,
9875) -> Result<()> {
9876    if (min_supported..=max_supported).contains(&version) {
9877        return Ok(());
9878    }
9879    Err(Error::NonDeterministicReplay(ReplayFailure::new(
9880        "version_marker_incompatible_range",
9881        (sequence != 0).then_some(sequence),
9882        Some(format!("{min_supported}..={max_supported}")),
9883        Some(format!("{change_id}:{version}")),
9884        "recorded workflow version is outside the range supported by current code",
9885    )))
9886}
9887
9888#[derive(Clone, Debug)]
9889struct ResumeSignal {
9890    signal_name: String,
9891    arguments: Vec<AvroValue>,
9892}
9893
9894const MAX_PARALLEL_OPERATIONS: usize = 1000;
9895
9896fn parallel_group_prefix(kind: &str) -> &'static str {
9897    match kind {
9898        "activity" => "parallel-activities",
9899        "child" => "parallel-children",
9900        "timer" => "parallel-timers",
9901        _ => "parallel-calls",
9902    }
9903}
9904
9905fn parallel_group_entry(
9906    base_sequence: u64,
9907    size: usize,
9908    index: usize,
9909    kind: &str,
9910) -> ParallelGroupMetadata {
9911    ParallelGroupMetadata {
9912        parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9913        parallel_group_kind: kind.to_string(),
9914        parallel_group_base_sequence: base_sequence,
9915        parallel_group_size: size,
9916        parallel_group_index: index,
9917        parallel_group_mode: None,
9918        selection_member_key: None,
9919        selection_member_index: None,
9920        selection_member_base_sequence: None,
9921        selection_member_size: None,
9922        selection_member_kind: None,
9923    }
9924}
9925
9926struct SelectionMemberMetadata {
9927    key: SelectionKey,
9928    index: usize,
9929    base_sequence: u64,
9930    size: usize,
9931    kind: String,
9932}
9933
9934fn selection_group_entry(
9935    base_sequence: u64,
9936    size: usize,
9937    index: usize,
9938    kind: &str,
9939    member: &SelectionMemberMetadata,
9940) -> ParallelGroupMetadata {
9941    ParallelGroupMetadata {
9942        parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9943        parallel_group_kind: kind.to_string(),
9944        parallel_group_base_sequence: base_sequence,
9945        parallel_group_size: size,
9946        parallel_group_index: index,
9947        parallel_group_mode: Some("select".to_string()),
9948        selection_member_key: Some(member.key.clone()),
9949        selection_member_index: Some(member.index),
9950        selection_member_base_sequence: Some(member.base_sequence),
9951        selection_member_size: Some(member.size),
9952        selection_member_kind: Some(member.kind.clone()),
9953    }
9954}
9955
9956fn apply_parallel_group_path(
9957    command: &mut serde_json::Map<String, Value>,
9958    path: &[ParallelGroupMetadata],
9959) {
9960    let Some(inner) = path.last() else {
9961        return;
9962    };
9963    command.insert(
9964        "parallel_group_id".to_string(),
9965        json!(inner.parallel_group_id),
9966    );
9967    command.insert(
9968        "parallel_group_kind".to_string(),
9969        json!(inner.parallel_group_kind),
9970    );
9971    command.insert(
9972        "parallel_group_base_sequence".to_string(),
9973        json!(inner.parallel_group_base_sequence),
9974    );
9975    command.insert(
9976        "parallel_group_size".to_string(),
9977        json!(inner.parallel_group_size),
9978    );
9979    command.insert(
9980        "parallel_group_index".to_string(),
9981        json!(inner.parallel_group_index),
9982    );
9983    if let Some(mode) = &inner.parallel_group_mode {
9984        command.insert("parallel_group_mode".to_string(), json!(mode));
9985    }
9986    if let Some(key) = &inner.selection_member_key {
9987        command.insert("selection_member_key".to_string(), json!(key));
9988    }
9989    if let Some(index) = inner.selection_member_index {
9990        command.insert("selection_member_index".to_string(), json!(index));
9991    }
9992    if let Some(base_sequence) = inner.selection_member_base_sequence {
9993        command.insert(
9994            "selection_member_base_sequence".to_string(),
9995            json!(base_sequence),
9996        );
9997    }
9998    if let Some(size) = inner.selection_member_size {
9999        command.insert("selection_member_size".to_string(), json!(size));
10000    }
10001    if let Some(kind) = &inner.selection_member_kind {
10002        command.insert("selection_member_kind".to_string(), json!(kind));
10003    }
10004    command.insert("parallel_group_path".to_string(), json!(path));
10005}
10006
10007fn ensure_parallel_path_matches(
10008    sequence: u64,
10009    recorded: Option<&[ParallelGroupMetadata]>,
10010    expected: &[ParallelGroupMetadata],
10011) -> Result<()> {
10012    match (recorded, expected.is_empty()) {
10013        (None, true) => Ok(()),
10014        (Some(recorded), false) if recorded == expected => Ok(()),
10015        (None, false) => Err(invalid_recorded_history(
10016            "parallel_group_metadata_missing",
10017            sequence,
10018            &serde_json::to_string(expected).unwrap_or_default(),
10019            "<missing>",
10020            "recorded parallel member is missing its durable group path",
10021        )),
10022        (Some(recorded), true) => Err(invalid_recorded_history(
10023            "parallel_group_shape_mismatch",
10024            sequence,
10025            "sequential command",
10026            &serde_json::to_string(recorded).unwrap_or_default(),
10027            "recorded command belonged to a parallel group but current code schedules it sequentially",
10028        )),
10029        (Some(recorded), false) => Err(invalid_recorded_history(
10030            "parallel_group_shape_mismatch",
10031            sequence,
10032            &serde_json::to_string(recorded).unwrap_or_default(),
10033            &serde_json::to_string(expected).unwrap_or_default(),
10034            "recorded parallel-group identity or path changed during replay",
10035        )),
10036    }
10037}
10038
10039#[derive(Clone, Debug)]
10040enum ParallelShape {
10041    Leaf,
10042    Group(Vec<ParallelShape>),
10043}
10044
10045struct ParallelDescriptor {
10046    operation: ParallelOperation,
10047    offset: usize,
10048    member_path: Vec<usize>,
10049    group_path: Vec<ParallelGroupMetadata>,
10050}
10051
10052fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
10053    operations
10054        .iter()
10055        .map(|operation| match operation {
10056            ParallelOperation::Group(children) => parallel_leaf_count(children),
10057            _ => 1,
10058        })
10059        .sum()
10060}
10061
10062fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
10063    match operation {
10064        ParallelOperation::Activity { .. } => Some("activity"),
10065        ParallelOperation::ChildWorkflow { .. } => Some("child"),
10066        ParallelOperation::Timer(_) => Some("timer"),
10067        ParallelOperation::Signal(_) => Some("signal"),
10068        ParallelOperation::Condition { .. } => Some("condition"),
10069        ParallelOperation::Group(children) => parallel_group_kind(children),
10070    }
10071}
10072
10073fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
10074    let mut kind = None;
10075    for operation in operations {
10076        let Some(operation_kind) = parallel_operation_kind(operation) else {
10077            continue;
10078        };
10079        match kind {
10080            None => kind = Some(operation_kind),
10081            Some(current) if current == operation_kind => {}
10082            Some(_) => return Some("mixed"),
10083        }
10084    }
10085    kind
10086}
10087
10088fn validate_parallel_operations(
10089    operations: &[ParallelOperation],
10090    member_path: &mut Vec<usize>,
10091    root: bool,
10092) -> Result<()> {
10093    let leaves = parallel_leaf_count(operations);
10094    if leaves > MAX_PARALLEL_OPERATIONS {
10095        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10096            reason: "fan_out_limit_exceeded",
10097            member_path: member_path.clone(),
10098            message: format!(
10099                "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10100            ),
10101        }));
10102    }
10103    if !root && operations.is_empty() {
10104        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10105            reason: "nested_group_empty",
10106            member_path: member_path.clone(),
10107            message: "a nested group must contain at least one durable leaf".to_string(),
10108        }));
10109    }
10110
10111    for (index, operation) in operations.iter().enumerate() {
10112        member_path.push(index);
10113        match operation {
10114            ParallelOperation::Activity {
10115                options, arguments, ..
10116            } => {
10117                options
10118                    .validate()
10119                    .map_err(|error| Error::InvalidActivityOptions(error))?;
10120                if let Err(error) = arguments {
10121                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10122                        reason: "arguments_invalid",
10123                        member_path: member_path.clone(),
10124                        message: error.to_string(),
10125                    }));
10126                }
10127            }
10128            ParallelOperation::ChildWorkflow {
10129                options, arguments, ..
10130            } => {
10131                validate_parallel_child_options(options)?;
10132                if let Err(error) = arguments {
10133                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10134                        reason: "arguments_invalid",
10135                        member_path: member_path.clone(),
10136                        message: error.to_string(),
10137                    }));
10138                }
10139            }
10140            ParallelOperation::Timer(duration)
10141                if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10142            {
10143                return Err(Error::TimerDurationOverflow);
10144            }
10145            ParallelOperation::Timer(_) => {}
10146            ParallelOperation::Signal(signal_name) => {
10147                validate_user_signal_name(signal_name)?;
10148                if signal_name.trim().is_empty() {
10149                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10150                        reason: "signal_name_empty",
10151                        member_path: member_path.clone(),
10152                        message: "signal wait name must not be empty".to_string(),
10153                    }));
10154                }
10155            }
10156            ParallelOperation::Condition { options, .. } => {
10157                options.validate()?;
10158            }
10159            ParallelOperation::Group(children) => {
10160                validate_parallel_operations(children, member_path, false)?;
10161            }
10162        }
10163        member_path.pop();
10164    }
10165    Ok(())
10166}
10167
10168fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10169    if options.task_queue.trim().is_empty() {
10170        return Err(Error::InvalidChildWorkflowOptions(
10171            "task_queue must not be empty".to_string(),
10172        ));
10173    }
10174    for (name, value) in [
10175        (
10176            "execution_timeout_seconds",
10177            options.execution_timeout_seconds,
10178        ),
10179        ("run_timeout_seconds", options.run_timeout_seconds),
10180    ] {
10181        if value == Some(0) {
10182            return Err(Error::InvalidChildWorkflowOptions(format!(
10183                "{name} must be at least 1"
10184            )));
10185        }
10186    }
10187    if options
10188        .retry_policy
10189        .as_ref()
10190        .is_some_and(|policy| policy.max_attempts == Some(0))
10191    {
10192        return Err(Error::InvalidChildWorkflowOptions(
10193            "retry_policy.max_attempts must be at least 1".to_string(),
10194        ));
10195    }
10196    Ok(())
10197}
10198
10199fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10200    ParallelShape::Group(
10201        operations
10202            .iter()
10203            .map(|operation| match operation {
10204                ParallelOperation::Group(children) => parallel_shape(children),
10205                _ => ParallelShape::Leaf,
10206            })
10207            .collect(),
10208    )
10209}
10210
10211fn parallel_descriptors(
10212    operations: Vec<ParallelOperation>,
10213    base_sequence: u64,
10214) -> Result<Vec<ParallelDescriptor>> {
10215    let size = parallel_leaf_count(&operations);
10216    let kind = parallel_group_kind(&operations).unwrap_or("activity");
10217    let mut descriptors = Vec::with_capacity(size);
10218    let mut cursor = 0;
10219
10220    for (index, operation) in operations.into_iter().enumerate() {
10221        match operation {
10222            ParallelOperation::Group(children) => {
10223                let child_base = base_sequence
10224                    .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10225                    .ok_or(Error::TimerDurationOverflow)?;
10226                for mut descriptor in parallel_descriptors(children, child_base)? {
10227                    let outer_index = cursor + descriptor.offset;
10228                    descriptor.group_path.insert(
10229                        0,
10230                        parallel_group_entry(base_sequence, size, outer_index, kind),
10231                    );
10232                    descriptor.member_path.insert(0, index);
10233                    descriptor.offset = outer_index;
10234                    descriptors.push(descriptor);
10235                }
10236                cursor = descriptors.len();
10237            }
10238            operation => {
10239                descriptors.push(ParallelDescriptor {
10240                    operation,
10241                    offset: cursor,
10242                    member_path: vec![index],
10243                    group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10244                });
10245                cursor += 1;
10246            }
10247        }
10248    }
10249    Ok(descriptors)
10250}
10251
10252enum ParallelLeafCall {
10253    Activity(ActivityCall),
10254    ChildWorkflow(ChildWorkflowCall),
10255    Timer(TimerCall),
10256    Signal(SignalCall),
10257    Condition(ConditionWaitCall),
10258}
10259
10260fn parallel_leaf_call(
10261    ctx: &WorkflowContext,
10262    operation: ParallelOperation,
10263    parallel_group_path: Vec<ParallelGroupMetadata>,
10264) -> ParallelLeafCall {
10265    match operation {
10266        ParallelOperation::Activity {
10267            activity_type,
10268            options,
10269            arguments,
10270        } => ParallelLeafCall::Activity(ActivityCall {
10271            ctx: ctx.clone(),
10272            activity_type,
10273            options,
10274            args: Some(arguments),
10275            scheduled: false,
10276            parallel_group_path,
10277        }),
10278        ParallelOperation::ChildWorkflow {
10279            workflow_type,
10280            options,
10281            arguments,
10282        } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10283            ctx: ctx.clone(),
10284            workflow_type,
10285            options,
10286            args: Some(arguments),
10287            scheduled: false,
10288            matched_pending: false,
10289            parallel_group_path,
10290        }),
10291        ParallelOperation::Timer(duration) => {
10292            let delay_seconds = duration
10293                .as_secs()
10294                .checked_add(u64::from(duration.subsec_nanos() > 0));
10295            ParallelLeafCall::Timer(TimerCall {
10296                ctx: ctx.clone(),
10297                delay_seconds,
10298                scheduled: false,
10299                matched_pending: false,
10300                parallel_group_path,
10301            })
10302        }
10303        ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10304            ctx: ctx.clone(),
10305            signal_name,
10306            runtime_reserved_allowed: false,
10307            opened_wait: false,
10308            matched_pending: false,
10309            parallel_group_path,
10310        }),
10311        ParallelOperation::Condition { options, predicate } => {
10312            ParallelLeafCall::Condition(ConditionWaitCall {
10313                ctx: ctx.clone(),
10314                options,
10315                predicate,
10316                occurrence_id: None,
10317                opened_wait: false,
10318                parallel_group_path,
10319            })
10320        }
10321        ParallelOperation::Group(_) => {
10322            unreachable!("parallel descriptors contain only durable leaves")
10323        }
10324    }
10325}
10326
10327impl ParallelLeafCall {
10328    fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10329        match self {
10330            Self::Activity(call) => Pin::new(call)
10331                .poll_avro_value(cx)
10332                .map_ok(ParallelAvroResult::Activity),
10333            Self::ChildWorkflow(call) => Pin::new(call)
10334                .poll_avro_value(cx)
10335                .map_ok(ParallelAvroResult::ChildWorkflow),
10336            Self::Timer(call) => Pin::new(call)
10337                .poll(cx)
10338                .map_ok(|()| ParallelAvroResult::Timer),
10339            Self::Signal(call) => Pin::new(call)
10340                .poll_avro_value(cx)
10341                .map_ok(ParallelAvroResult::Signal),
10342            Self::Condition(call) => Pin::new(call)
10343                .poll(cx)
10344                .map_ok(ParallelAvroResult::Condition),
10345        }
10346    }
10347}
10348
10349struct ParallelLeaf {
10350    call: ParallelLeafCall,
10351    member_path: Vec<usize>,
10352    group_path: Vec<ParallelGroupMetadata>,
10353    result: Option<ParallelAvroResult>,
10354}
10355
10356/// Future returned by [`WorkflowContext::parallel`].
10357pub struct ParallelCall {
10358    ctx: WorkflowContext,
10359    operations: Option<Vec<ParallelOperation>>,
10360    shape: Option<ParallelShape>,
10361    leaves: Vec<ParallelLeaf>,
10362}
10363
10364impl ParallelCall {
10365    fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10366        Self {
10367            ctx,
10368            operations: Some(operations),
10369            shape: None,
10370            leaves: Vec::new(),
10371        }
10372    }
10373
10374    fn initialize(&mut self) -> Result<()> {
10375        let operations = self.operations.take().unwrap_or_default();
10376        validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10377        self.shape = Some(parallel_shape(&operations));
10378        if operations.is_empty() {
10379            return Ok(());
10380        }
10381
10382        let base_sequence = {
10383            let state = self
10384                .ctx
10385                .state
10386                .lock()
10387                .map_err(|_| Error::WorkflowStatePoisoned)?;
10388            if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10389                recorded.sequence()
10390            } else {
10391                let last = state
10392                    .recorded_commands
10393                    .last()
10394                    .map(RecordedCommand::sequence)
10395                    .unwrap_or(0);
10396                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10397                    .and_then(|sequence| sequence.checked_add(1))
10398                    .ok_or_else(|| {
10399                        Error::InvalidParallelGroup(ParallelGroupError {
10400                            reason: "sequence_overflow",
10401                            member_path: Vec::new(),
10402                            message: "parallel group sequence identity overflowed u64".to_string(),
10403                        })
10404                    })?
10405            }
10406        };
10407
10408        self.leaves = parallel_descriptors(operations, base_sequence)?
10409            .into_iter()
10410            .map(|descriptor| {
10411                let call = parallel_leaf_call(
10412                    &self.ctx,
10413                    descriptor.operation,
10414                    descriptor.group_path.clone(),
10415                );
10416                ParallelLeaf {
10417                    call,
10418                    member_path: descriptor.member_path,
10419                    group_path: descriptor.group_path,
10420                    result: None,
10421                }
10422            })
10423            .collect();
10424        Ok(())
10425    }
10426
10427    fn poll_avro_value(
10428        mut self: Pin<&mut Self>,
10429        cx: &mut TaskContext<'_>,
10430    ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10431        if self.operations.is_some() {
10432            if let Err(error) = self.initialize() {
10433                return Poll::Ready(Err(error));
10434            }
10435        }
10436        if self.leaves.is_empty() {
10437            return Poll::Ready(Ok(Vec::new()));
10438        }
10439
10440        let mut failures = Vec::new();
10441        let mut pending = false;
10442        for (index, leaf) in self.leaves.iter_mut().enumerate() {
10443            if leaf.result.is_some() {
10444                continue;
10445            }
10446            match leaf.call.poll_avro_value(cx) {
10447                Poll::Ready(Ok(result)) => leaf.result = Some(result),
10448                Poll::Ready(Err(error)) => failures.push((index, error)),
10449                Poll::Pending => pending = true,
10450            }
10451        }
10452
10453        if !failures.is_empty() {
10454            if let Some(position) = failures
10455                .iter()
10456                .position(|(_, error)| workflow_task_integrity_error(error))
10457            {
10458                return Poll::Ready(Err(failures.remove(position).1));
10459            }
10460            failures.sort_by_key(|(index, _)| *index);
10461            let (failed_index, cause) = failures.remove(0);
10462            let failed = &self.leaves[failed_index];
10463            let completed = self
10464                .leaves
10465                .iter()
10466                .filter_map(|leaf| {
10467                    leaf.result
10468                        .clone()
10469                        .and_then(|result| result.into_json_result().ok())
10470                        .map(|result| ParallelCompletion {
10471                            member_path: leaf.member_path.clone(),
10472                            result,
10473                        })
10474                })
10475                .collect();
10476            let group_id = failed
10477                .group_path
10478                .first()
10479                .map(|entry| entry.parallel_group_id.clone())
10480                .unwrap_or_default();
10481            return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10482                group_id,
10483                member_path: failed.member_path.clone(),
10484                group_path: failed.group_path.clone(),
10485                completed,
10486                cause: Box::new(cause),
10487            })));
10488        }
10489        if pending {
10490            return Poll::Pending;
10491        }
10492
10493        let mut flat_results = self
10494            .leaves
10495            .iter_mut()
10496            .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10497            .collect::<Vec<_>>()
10498            .into_iter();
10499        let results = parallel_results_for_shape(
10500            self.shape.as_ref().expect("initialized parallel shape"),
10501            &mut flat_results,
10502        );
10503        Poll::Ready(Ok(match results {
10504            ParallelAvroResult::Group(results) => results,
10505            ParallelAvroResult::Activity(_)
10506            | ParallelAvroResult::ChildWorkflow(_)
10507            | ParallelAvroResult::Timer
10508            | ParallelAvroResult::Signal(_)
10509            | ParallelAvroResult::Condition(_) => {
10510                unreachable!("root parallel shape is a group")
10511            }
10512        }))
10513    }
10514}
10515
10516fn parallel_results_for_shape(
10517    shape: &ParallelShape,
10518    flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10519) -> ParallelAvroResult {
10520    match shape {
10521        ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10522        ParallelShape::Group(children) => ParallelAvroResult::Group(
10523            children
10524                .iter()
10525                .map(|child| parallel_results_for_shape(child, flat_results))
10526                .collect(),
10527        ),
10528    }
10529}
10530
10531impl Future for ParallelCall {
10532    type Output = Result<Vec<ParallelResult>>;
10533
10534    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10535        self.poll_avro_value(cx)
10536            .map_ok(|results| {
10537                results
10538                    .into_iter()
10539                    .map(ParallelAvroResult::into_json_result)
10540                    .collect::<Result<Vec<_>>>()
10541            })
10542            .map_ok(|result| result)
10543            .flatten_result()
10544    }
10545}
10546
10547#[derive(Clone, Debug)]
10548struct SelectionMemberPlan {
10549    key: SelectionKey,
10550    index: usize,
10551    base_sequence: u64,
10552    size: usize,
10553    kind: String,
10554    shape: ParallelShape,
10555    leaf_start: usize,
10556}
10557
10558fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10559    match operation {
10560        ParallelOperation::Activity { .. } => "activity",
10561        ParallelOperation::ChildWorkflow { .. } => "child",
10562        ParallelOperation::Timer(_) => "timer",
10563        ParallelOperation::Signal(_) => "signal",
10564        ParallelOperation::Condition { .. } => "condition",
10565        ParallelOperation::Group(_) => "group",
10566    }
10567}
10568
10569fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10570    match operation {
10571        ParallelOperation::Group(children) => parallel_shape(children),
10572        _ => ParallelShape::Leaf,
10573    }
10574}
10575
10576fn selection_descriptors(
10577    operations: Vec<(SelectionKey, ParallelOperation)>,
10578    base_sequence: u64,
10579) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10580    if operations.is_empty() {
10581        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10582            reason: "selection_empty",
10583            member_path: Vec::new(),
10584            message: "durable selection requires at least one operation".to_string(),
10585        }));
10586    }
10587    let operation_refs = operations
10588        .iter()
10589        .map(|(_, operation)| operation)
10590        .collect::<Vec<_>>();
10591    let total_size = operation_refs
10592        .iter()
10593        .map(|operation| match operation {
10594            ParallelOperation::Group(children) => parallel_leaf_count(children),
10595            _ => 1,
10596        })
10597        .sum::<usize>();
10598    if total_size > MAX_PARALLEL_OPERATIONS {
10599        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10600            reason: "fan_out_limit_exceeded",
10601            member_path: Vec::new(),
10602            message: format!(
10603                "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10604            ),
10605        }));
10606    }
10607    let group_kind = {
10608        let mut kind = None;
10609        for operation in &operation_refs {
10610            let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10611            match kind {
10612                None => kind = Some(operation_kind),
10613                Some(current) if current == operation_kind => {}
10614                Some(_) => {
10615                    kind = Some("mixed");
10616                    break;
10617                }
10618            }
10619        }
10620        kind.unwrap_or("mixed")
10621    };
10622
10623    let mut descriptors = Vec::with_capacity(total_size);
10624    let mut members = Vec::with_capacity(operations.len());
10625    let mut cursor = 0usize;
10626    let mut seen_keys: Vec<SelectionKey> = Vec::new();
10627    for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10628        if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10629            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10630                reason: "selection_key_invalid",
10631                member_path: vec![member_index],
10632                message: "selection member keys must be non-empty strings or non-negative integers"
10633                    .to_string(),
10634            }));
10635        }
10636        if seen_keys.contains(&key) {
10637            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10638                reason: "selection_key_duplicate",
10639                member_path: vec![member_index],
10640                message: format!("selection member key {key:?} is duplicated"),
10641            }));
10642        }
10643        seen_keys.push(key.clone());
10644        let member_size = match &operation {
10645            ParallelOperation::Group(children) => parallel_leaf_count(children),
10646            _ => 1,
10647        };
10648        if member_size == 0 {
10649            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10650                reason: "selection_member_empty",
10651                member_path: vec![member_index],
10652                message: "a selection member must contain at least one durable leaf".to_string(),
10653            }));
10654        }
10655        let member_base = base_sequence
10656            .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10657            .ok_or(Error::TimerDurationOverflow)?;
10658        let member_kind = selection_operation_kind(&operation).to_string();
10659        let member_shape = selection_operation_shape(&operation);
10660        let leaf_start = descriptors.len();
10661        match operation {
10662            ParallelOperation::Group(children) => {
10663                validate_parallel_operations(&children, &mut vec![member_index], false)?;
10664                for mut descriptor in parallel_descriptors(children, member_base)? {
10665                    let flat_index = cursor + descriptor.offset;
10666                    descriptor.group_path.insert(
10667                        0,
10668                        selection_group_entry(
10669                            base_sequence,
10670                            total_size,
10671                            flat_index,
10672                            group_kind,
10673                            &SelectionMemberMetadata {
10674                                key: key.clone(),
10675                                index: member_index,
10676                                base_sequence: member_base,
10677                                size: member_size,
10678                                kind: member_kind.clone(),
10679                            },
10680                        ),
10681                    );
10682                    descriptor.member_path.insert(0, member_index);
10683                    descriptor.offset = flat_index;
10684                    descriptors.push(descriptor);
10685                }
10686            }
10687            operation => {
10688                validate_parallel_operations(
10689                    std::slice::from_ref(&operation),
10690                    &mut Vec::new(),
10691                    true,
10692                )?;
10693                descriptors.push(ParallelDescriptor {
10694                    operation,
10695                    offset: cursor,
10696                    member_path: vec![member_index],
10697                    group_path: vec![selection_group_entry(
10698                        base_sequence,
10699                        total_size,
10700                        cursor,
10701                        group_kind,
10702                        &SelectionMemberMetadata {
10703                            key: key.clone(),
10704                            index: member_index,
10705                            base_sequence: member_base,
10706                            size: member_size,
10707                            kind: member_kind.clone(),
10708                        },
10709                    )],
10710                });
10711            }
10712        }
10713        members.push(SelectionMemberPlan {
10714            key,
10715            index: member_index,
10716            base_sequence: member_base,
10717            size: member_size,
10718            kind: member_kind,
10719            shape: member_shape,
10720            leaf_start,
10721        });
10722        cursor += member_size;
10723    }
10724    Ok((descriptors, members))
10725}
10726
10727struct SelectionLeaf {
10728    call: ParallelLeafCall,
10729    outcome: Option<Result<ParallelAvroResult>>,
10730}
10731
10732/// Stable reference to one member of a durable selection group.
10733#[derive(Clone)]
10734pub struct DurableOperationHandle {
10735    ctx: WorkflowContext,
10736    pub key: SelectionKey,
10737    pub index: usize,
10738    pub kind: String,
10739    pub identity: String,
10740    pub base_sequence: u64,
10741    pub size: usize,
10742    pub selection_group_id: String,
10743    shape: ParallelShape,
10744}
10745
10746impl std::fmt::Debug for DurableOperationHandle {
10747    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10748        formatter
10749            .debug_struct("DurableOperationHandle")
10750            .field("key", &self.key)
10751            .field("index", &self.index)
10752            .field("kind", &self.kind)
10753            .field("identity", &self.identity)
10754            .field("base_sequence", &self.base_sequence)
10755            .field("size", &self.size)
10756            .field("selection_group_id", &self.selection_group_id)
10757            .finish()
10758    }
10759}
10760
10761impl DurableOperationHandle {
10762    /// Await this member after another member has already won the selection.
10763    pub fn await_result(&self) -> DurableOperationAwaitCall {
10764        DurableOperationAwaitCall {
10765            handle: self.clone(),
10766        }
10767    }
10768
10769    /// Request cancellation without affecting siblings. The future resolves
10770    /// to unit; only `SelectionOperationCancelled` history proves cancellation
10771    /// beat a concurrently committed terminal result.
10772    pub fn cancel(&self) -> CancelDurableOperationCall {
10773        CancelDurableOperationCall {
10774            handle: self.clone(),
10775            emitted: false,
10776        }
10777    }
10778}
10779
10780/// The one winner committed for a durable selection group.
10781#[derive(Debug)]
10782pub struct SelectionResult {
10783    pub key: SelectionKey,
10784    pub index: usize,
10785    pub kind: String,
10786    pub identity: String,
10787    pub value: Option<ParallelResult>,
10788    pub failure: Option<Error>,
10789    pub winner: DurableOperationHandle,
10790    pub handles: Vec<DurableOperationHandle>,
10791}
10792
10793impl SelectionResult {
10794    pub fn succeeded(&self) -> bool {
10795        self.failure.is_none()
10796    }
10797
10798    pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10799        self.handles.iter().find(|handle| &handle.key == key)
10800    }
10801
10802    pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10803        self.handles
10804            .iter()
10805            .filter(|handle| handle.index != self.index)
10806            .collect()
10807    }
10808
10809    pub fn into_result(self) -> Result<ParallelResult> {
10810        match (self.value, self.failure) {
10811            (Some(value), None) => Ok(value),
10812            (_, Some(error)) => Err(error),
10813            _ => Err(Error::WorkerLoop(
10814                "selection result contained neither a value nor a failure".to_string(),
10815            )),
10816        }
10817    }
10818}
10819
10820/// Future returned by [`WorkflowContext::select`].
10821pub struct SelectCall {
10822    ctx: WorkflowContext,
10823    operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10824    members: Vec<SelectionMemberPlan>,
10825    leaves: Vec<SelectionLeaf>,
10826    group_id: Option<String>,
10827}
10828
10829impl SelectCall {
10830    fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10831        Self {
10832            ctx,
10833            operations: Some(operations),
10834            members: Vec::new(),
10835            leaves: Vec::new(),
10836            group_id: None,
10837        }
10838    }
10839
10840    fn initialize(&mut self) -> Result<()> {
10841        let operations = self.operations.take().unwrap_or_default();
10842        let base_sequence = {
10843            let state = self
10844                .ctx
10845                .state
10846                .lock()
10847                .map_err(|_| Error::WorkflowStatePoisoned)?;
10848            if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10849                marker.selection_group_base_sequence
10850            } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10851                recorded.sequence()
10852            } else {
10853                let last = state
10854                    .recorded_commands
10855                    .last()
10856                    .map(RecordedCommand::sequence)
10857                    .unwrap_or(0);
10858                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10859                    .and_then(|sequence| sequence.checked_add(1))
10860                    .ok_or(Error::TimerDurationOverflow)?
10861            }
10862        };
10863        let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10864        let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10865        self.leaves = descriptors
10866            .into_iter()
10867            .map(|descriptor| SelectionLeaf {
10868                call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10869                outcome: None,
10870            })
10871            .collect();
10872        self.members = members;
10873        self.group_id = Some(group_id);
10874        Ok(())
10875    }
10876}
10877
10878impl Future for SelectCall {
10879    type Output = Result<SelectionResult>;
10880
10881    fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10882        if self.operations.is_some() {
10883            if let Err(error) = self.initialize() {
10884                return Poll::Ready(Err(error));
10885            }
10886        }
10887
10888        for leaf in &mut self.leaves {
10889            if leaf.outcome.is_some() {
10890                continue;
10891            }
10892            if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10893                if outcome
10894                    .as_ref()
10895                    .err()
10896                    .is_some_and(workflow_task_integrity_error)
10897                {
10898                    return Poll::Ready(outcome.map(|_| unreachable!()));
10899                }
10900                leaf.outcome = Some(outcome);
10901            }
10902        }
10903
10904        let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10905        let selection_member_range = self
10906            .members
10907            .first()
10908            .map(|member| member.base_sequence)
10909            .zip(self.leaves.len().try_into().ok())
10910            .map(|(base_sequence, size): (u64, u64)| {
10911                base_sequence..base_sequence.saturating_add(size)
10912            });
10913        let marker = {
10914            let mut state = match self.ctx.state.lock() {
10915                Ok(state) => state,
10916                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10917            };
10918            let marker = state
10919                .selection_markers
10920                .get(state.selection_marker_cursor)
10921                .cloned();
10922            if marker.is_none()
10923                && all_members_terminal
10924                && selection_member_range.as_ref().is_some_and(|member_range| {
10925                    state
10926                        .recorded_commands
10927                        .iter()
10928                        .any(|command| member_range.contains(&command.sequence()))
10929                })
10930            {
10931                // Terminal member history can be committed before the server's
10932                // SelectionResolved marker becomes visible to this task. That
10933                // marker is the durable barrier the selection is still waiting
10934                // on, so an otherwise commandless replay remains legitimately
10935                // pending instead of failing as an untracked yield.
10936                state.matched_recorded_pending = true;
10937            }
10938            marker
10939        };
10940        let Some(marker) = marker else {
10941            return Poll::Pending;
10942        };
10943        if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10944            || marker.selection_group_size != self.leaves.len()
10945            || self.members.first().map(|member| member.base_sequence)
10946                != Some(marker.selection_group_base_sequence)
10947        {
10948            return Poll::Ready(Err(invalid_recorded_history(
10949                "selection_group_shape_mismatch",
10950                marker.selection_group_base_sequence,
10951                self.group_id
10952                    .as_deref()
10953                    .unwrap_or("initialized selection group"),
10954                &marker.selection_group_id,
10955                "recorded selection group differs from current workflow code",
10956            )));
10957        }
10958        let Some(member_position) = self.members.iter().position(|member| {
10959            member.key == marker.member_key
10960                && member.index == marker.member_index
10961                && member.base_sequence == marker.member_base_sequence
10962                && member.size == marker.member_size
10963                && member.kind == marker.operation_kind
10964        }) else {
10965            return Poll::Ready(Err(invalid_recorded_history(
10966                "selection_member_shape_mismatch",
10967                marker.member_base_sequence,
10968                "winner member matching current workflow code",
10969                &format!("{:?}", marker.member_key),
10970                "recorded selection winner differs from the authored member identity",
10971            )));
10972        };
10973        let member = self.members[member_position].clone();
10974        let (handles, resolution_sequence) = {
10975            let mut state = match self.ctx.state.lock() {
10976                Ok(state) => state,
10977                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10978            };
10979            let identities = self
10980                .members
10981                .iter()
10982                .map(|candidate| {
10983                    selection_operation_identity(
10984                        &state,
10985                        &candidate.kind,
10986                        candidate.base_sequence,
10987                        candidate.size,
10988                    )
10989                })
10990                .collect::<Vec<_>>();
10991            if let Some((position, missing)) = identities
10992                .iter()
10993                .enumerate()
10994                .find(|(_, identity)| identity.is_empty())
10995                .map(|(position, identity)| (position, identity.clone()))
10996            {
10997                let candidate = &self.members[position];
10998                return Poll::Ready(Err(invalid_recorded_history(
10999                    "selection_operation_identity_missing",
11000                    candidate.base_sequence,
11001                    &format!(
11002                        "durable {} resource identity from scheduled/open history",
11003                        candidate.kind
11004                    ),
11005                    &missing,
11006                    "selection member history is missing its canonical durable identity",
11007                )));
11008            }
11009            let expected_winner_identity = &identities[member_position];
11010            let resolution_sequence = match validated_selection_resolution_sequence(
11011                &state,
11012                &marker,
11013                &member,
11014                expected_winner_identity,
11015            ) {
11016                Ok(sequence) => sequence,
11017                Err(error) => return Poll::Ready(Err(error)),
11018            };
11019            let handles = self
11020                .members
11021                .iter()
11022                .zip(identities)
11023                .map(|(member, identity)| DurableOperationHandle {
11024                    ctx: self.ctx.clone(),
11025                    key: member.key.clone(),
11026                    index: member.index,
11027                    kind: member.kind.clone(),
11028                    identity,
11029                    base_sequence: member.base_sequence,
11030                    size: member.size,
11031                    selection_group_id: marker.selection_group_id.clone(),
11032                    shape: member.shape.clone(),
11033                })
11034                .collect::<Vec<_>>();
11035            if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
11036                return Poll::Ready(Err(error));
11037            }
11038            state.selection_marker_cursor += 1;
11039            (handles, resolution_sequence)
11040        };
11041
11042        let mut winner_failure = None;
11043        let mut flat_results = Vec::with_capacity(member.size);
11044        if marker.outcome == "failed" {
11045            let resolution_offset = match resolution_sequence
11046                .checked_sub(member.base_sequence)
11047                .and_then(|offset| usize::try_from(offset).ok())
11048            {
11049                Some(offset) if offset < member.size => offset,
11050                _ => {
11051                    return Poll::Ready(Err(invalid_recorded_history(
11052                        "selection_resolution_event_mismatch",
11053                        member.base_sequence,
11054                        "failure event within selected member bounds",
11055                        &resolution_sequence.to_string(),
11056                        "selection failure event is outside the authored member",
11057                    )))
11058                }
11059            };
11060            let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
11061            match leaf.outcome.take() {
11062                Some(Err(error)) => winner_failure = Some(error),
11063                _ => {
11064                    return Poll::Ready(Err(invalid_recorded_history(
11065                        "selection_winner_outcome_mismatch",
11066                        member.base_sequence,
11067                        "exact failed terminal history referenced by SelectionResolved",
11068                        "missing or successful resolution event",
11069                        "selection winner marker disagrees with terminal operation history",
11070                    )))
11071                }
11072            }
11073        } else {
11074            for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
11075                match leaf.outcome.take() {
11076                    Some(Ok(result)) => flat_results.push(result),
11077                    Some(Err(_)) => {
11078                        return Poll::Ready(Err(invalid_recorded_history(
11079                            "selection_winner_outcome_mismatch",
11080                            member.base_sequence,
11081                            "fully completed nested selection member",
11082                            "failed durable leaf",
11083                            "completed selection winner contains a failed leaf",
11084                        )))
11085                    }
11086                    None => {
11087                        return Poll::Ready(Err(invalid_recorded_history(
11088                            "selection_winner_unresolved",
11089                            member.base_sequence,
11090                            "terminal history for every completed winner leaf",
11091                            "pending member history",
11092                            "completed SelectionResolved member has an unfinished durable barrier",
11093                        )))
11094                    }
11095                }
11096            }
11097        }
11098        let value = if winner_failure.is_none() {
11099            let mut flat_results = flat_results.into_iter();
11100            let value = parallel_results_for_shape(&member.shape, &mut flat_results);
11101            match value.into_json_result() {
11102                Ok(value) => Some(value),
11103                Err(error) => return Poll::Ready(Err(error)),
11104            }
11105        } else {
11106            None
11107        };
11108        let winner = handles[member_position].clone();
11109        Poll::Ready(Ok(SelectionResult {
11110            key: winner.key.clone(),
11111            index: winner.index,
11112            kind: winner.kind.clone(),
11113            identity: winner.identity.clone(),
11114            value,
11115            failure: winner_failure,
11116            winner,
11117            handles,
11118        }))
11119    }
11120}
11121
11122fn selection_operation_identity(
11123    state: &WorkflowState,
11124    kind: &str,
11125    base_sequence: u64,
11126    size: usize,
11127) -> String {
11128    if kind == "group" {
11129        return format!("group:{base_sequence}:{size}");
11130    }
11131    let fields: &[&str] = match kind {
11132        "activity" => &["activity_execution_id"],
11133        "child" => &["child_workflow_run_id"],
11134        "timer" => &["timer_id"],
11135        "signal" => &["signal_wait_id"],
11136        "condition" => &["condition_wait_id"],
11137        _ => &[],
11138    };
11139    for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11140        for event in state
11141            .history_events
11142            .iter()
11143            .filter(|event| durable_event_sequence(event) == Some(sequence))
11144        {
11145            for field in fields {
11146                if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11147                    if !identity.is_empty() {
11148                        return identity.to_string();
11149                    }
11150                }
11151            }
11152        }
11153    }
11154    String::new()
11155}
11156
11157fn validated_selection_resolution_sequence(
11158    state: &WorkflowState,
11159    marker: &SelectionMarker,
11160    member: &SelectionMemberPlan,
11161    expected_identity: &str,
11162) -> Result<u64> {
11163    if expected_identity.is_empty() {
11164        return Err(invalid_recorded_history(
11165            "selection_operation_identity_missing",
11166            member.base_sequence,
11167            &format!(
11168                "durable {} resource identity from scheduled/open history",
11169                member.kind
11170            ),
11171            "missing operation identity",
11172            "selection member history is missing its canonical durable identity",
11173        ));
11174    }
11175    if marker.operation_identity != expected_identity {
11176        return Err(invalid_recorded_history(
11177            "selection_operation_identity_mismatch",
11178            member.base_sequence,
11179            expected_identity,
11180            &marker.operation_identity,
11181            "selection winner identity does not match durable scheduled/open history",
11182        ));
11183    }
11184
11185    let failure_types = [
11186        "ActivityFailed",
11187        "ActivityCancelled",
11188        "ActivityTimedOut",
11189        "ChildRunFailed",
11190        "ChildRunCancelled",
11191        "ChildRunTerminated",
11192    ];
11193    let success_types = [
11194        "ActivityCompleted",
11195        "ChildRunCompleted",
11196        "TimerFired",
11197        "SignalApplied",
11198        "ConditionWaitSatisfied",
11199        "ConditionWaitTimedOut",
11200    ];
11201    let terminal_types: &[&str] = if marker.outcome == "failed" {
11202        &failure_types
11203    } else {
11204        &success_types
11205    };
11206    let mut candidates = Vec::new();
11207    for event in state.history_events.iter() {
11208        let Some(sequence) = durable_event_sequence(event) else {
11209            continue;
11210        };
11211        if sequence < member.base_sequence
11212            || sequence >= member.base_sequence.saturating_add(member.size as u64)
11213            || !terminal_types.contains(&event.event_type.as_str())
11214        {
11215            continue;
11216        }
11217        let event_id = event
11218            .raw
11219            .get("id")
11220            .or_else(|| event.raw.get("event_id"))
11221            .and_then(Value::as_str)
11222            .filter(|value| !value.is_empty())
11223            .ok_or_else(|| {
11224                invalid_recorded_history(
11225                    "selection_resolution_event_id_missing",
11226                    member.base_sequence,
11227                    "terminal selection history with a durable event id",
11228                    &event.payload.to_string(),
11229                    "selection terminal history cannot be bound to its winner marker",
11230                )
11231            })?;
11232        candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11233    }
11234    let resolution = if marker.outcome == "failed" {
11235        candidates.first()
11236    } else {
11237        candidates.last()
11238    };
11239    let Some((event_id, event_type, sequence)) = resolution else {
11240        return Err(invalid_recorded_history(
11241            "selection_resolution_event_missing",
11242            member.base_sequence,
11243            "terminal history for the selected member",
11244            &format!("{:?}", marker.member_key),
11245            "selection winner marker has no matching durable terminal event",
11246        ));
11247    };
11248    if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11249        return Err(invalid_recorded_history(
11250            "selection_resolution_event_mismatch",
11251            member.base_sequence,
11252            &format!("{event_type}:{event_id}"),
11253            &format!(
11254                "{}:{}",
11255                marker.resolution_event_type, marker.resolution_event_id
11256            ),
11257            "selection winner marker does not reference the event that made its member terminal",
11258        ));
11259    }
11260    Ok(*sequence)
11261}
11262
11263fn recorded_selection_member_outcome(
11264    state: &WorkflowState,
11265    handle: &DurableOperationHandle,
11266) -> Result<Option<ParallelResult>> {
11267    for event in state.history_events.iter() {
11268        let Some(sequence) = durable_event_sequence(event) else {
11269            continue;
11270        };
11271        if sequence < handle.base_sequence
11272            || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11273            || !matches!(
11274                event.event_type.as_str(),
11275                "ActivityFailed"
11276                    | "ActivityCancelled"
11277                    | "ActivityTimedOut"
11278                    | "ChildRunFailed"
11279                    | "ChildRunCancelled"
11280                    | "ChildRunTerminated"
11281            )
11282        {
11283            continue;
11284        }
11285        let Some(command) = state
11286            .recorded_commands
11287            .iter()
11288            .find(|command| command.sequence() == sequence)
11289        else {
11290            continue;
11291        };
11292        match command {
11293            RecordedCommand::Activity {
11294                outcome: Some(Err(failure)),
11295                ..
11296            } => return Err(Error::ActivityFailed(failure.clone())),
11297            RecordedCommand::ChildWorkflow {
11298                outcome: Some(Err(failure)),
11299                ..
11300            } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11301            _ => {}
11302        }
11303    }
11304
11305    let mut results = Vec::with_capacity(handle.size);
11306    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11307        let Some(command) = state
11308            .recorded_commands
11309            .iter()
11310            .find(|command| command.sequence() == sequence)
11311        else {
11312            return Ok(None);
11313        };
11314        let result = match command {
11315            RecordedCommand::Activity { outcome, .. } => match outcome {
11316                Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11317                Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11318                None => return Ok(None),
11319            },
11320            RecordedCommand::Timer { fired, .. } => {
11321                if !fired {
11322                    return Ok(None);
11323                }
11324                ParallelAvroResult::Timer
11325            }
11326            RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11327                Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11328                Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11329                None => return Ok(None),
11330            },
11331            RecordedCommand::SignalWait { value, .. } => match value {
11332                Some(value) => ParallelAvroResult::Signal(value.clone()),
11333                None => return Ok(None),
11334            },
11335            RecordedCommand::ConditionWait { result, .. } => match result {
11336                Some(result) => ParallelAvroResult::Condition(*result),
11337                None => return Ok(None),
11338            },
11339            other => {
11340                return Err(command_mismatch(
11341                    other,
11342                    format!("selected {} member", handle.kind),
11343                ))
11344            }
11345        };
11346        results.push(result);
11347    }
11348    let mut results = results.into_iter();
11349    parallel_results_for_shape(&handle.shape, &mut results)
11350        .into_json_result()
11351        .map(Some)
11352}
11353
11354fn recorded_selection_member_is_terminal(
11355    state: &WorkflowState,
11356    handle: &DurableOperationHandle,
11357) -> bool {
11358    let mut completed = 0usize;
11359    let mut all_completed = true;
11360    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11361        let Some(command) = state
11362            .recorded_commands
11363            .iter()
11364            .find(|command| command.sequence() == sequence)
11365        else {
11366            all_completed = false;
11367            continue;
11368        };
11369        let terminal = match command {
11370            RecordedCommand::Activity {
11371                outcome: Some(Err(_)),
11372                ..
11373            }
11374            | RecordedCommand::ChildWorkflow {
11375                outcome: Some(Err(_)),
11376                ..
11377            } => return true,
11378            RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11379            RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11380            RecordedCommand::Timer { fired, .. } => *fired,
11381            RecordedCommand::SignalWait { value, .. } => value.is_some(),
11382            RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11383            RecordedCommand::SearchAttributes { .. }
11384            | RecordedCommand::SideEffect { .. }
11385            | RecordedCommand::VersionMarker { .. }
11386            | RecordedCommand::Memo { .. } => false,
11387        };
11388        if !terminal {
11389            all_completed = false;
11390            continue;
11391        }
11392        completed += 1;
11393    }
11394    all_completed && completed == handle.size
11395}
11396
11397fn selection_cancellation_for_handle(
11398    state: &WorkflowState,
11399    handle: &DurableOperationHandle,
11400) -> Result<bool> {
11401    let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11402        recorded.selection_group_id == handle.selection_group_id
11403            && recorded.member_base_sequence == handle.base_sequence
11404    }) else {
11405        return Ok(false);
11406    };
11407    validate_selection_cancellation_marker(marker, handle)?;
11408    Ok(true)
11409}
11410
11411fn validate_selection_cancellations_for_handles(
11412    state: &WorkflowState,
11413    handles: &[DurableOperationHandle],
11414) -> Result<()> {
11415    let Some(group_id) = handles
11416        .first()
11417        .map(|handle| handle.selection_group_id.as_str())
11418    else {
11419        return Ok(());
11420    };
11421    for marker in state
11422        .cancelled_selection_members
11423        .iter()
11424        .filter(|marker| marker.selection_group_id == group_id)
11425    {
11426        let Some(handle) = handles
11427            .iter()
11428            .find(|handle| handle.base_sequence == marker.member_base_sequence)
11429        else {
11430            return Err(invalid_recorded_history(
11431                "selection_cancellation_member_mismatch",
11432                marker.member_base_sequence,
11433                "SelectionOperationCancelled matching an authored selection handle",
11434                &format!("{marker:?}"),
11435                "selection cancellation member base does not name an authored member",
11436            ));
11437        };
11438        validate_selection_cancellation_marker(marker, handle)?;
11439    }
11440    Ok(())
11441}
11442
11443fn validate_selection_cancellation_marker(
11444    marker: &SelectionCancellation,
11445    handle: &DurableOperationHandle,
11446) -> Result<()> {
11447    if marker.selection_group_id != handle.selection_group_id
11448        || marker.member_key != handle.key
11449        || marker.member_index != handle.index
11450        || marker.member_base_sequence != handle.base_sequence
11451        || marker.member_size != handle.size
11452        || marker.operation_kind != handle.kind
11453        || marker.operation_identity != handle.identity
11454    {
11455        return Err(invalid_recorded_history(
11456            "selection_cancellation_member_mismatch",
11457            handle.base_sequence,
11458            "SelectionOperationCancelled matching the authored selection handle",
11459            &format!("{marker:?}"),
11460            "selection cancellation history targets different authored member metadata",
11461        ));
11462    }
11463    Ok(())
11464}
11465
11466/// Future returned by [`DurableOperationHandle::await_result`].
11467pub struct DurableOperationAwaitCall {
11468    handle: DurableOperationHandle,
11469}
11470
11471impl Future for DurableOperationAwaitCall {
11472    type Output = Result<ParallelResult>;
11473
11474    fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11475        let state = match self.handle.ctx.state.lock() {
11476            Ok(state) => state,
11477            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11478        };
11479        match selection_cancellation_for_handle(&state, &self.handle) {
11480            Err(error) => return Poll::Ready(Err(error)),
11481            Ok(false) => {}
11482            Ok(true) => {
11483                return Poll::Ready(Err(Error::DurableOperationCancelled(
11484                    DurableOperationCancelled {
11485                        selection_group_id: self.handle.selection_group_id.clone(),
11486                        member_key: self.handle.key.clone(),
11487                        member_index: self.handle.index,
11488                        operation_kind: self.handle.kind.clone(),
11489                        operation_identity: self.handle.identity.clone(),
11490                    },
11491                )));
11492            }
11493        }
11494        match recorded_selection_member_outcome(&state, &self.handle) {
11495            Ok(Some(result)) => Poll::Ready(Ok(result)),
11496            Ok(None) => Poll::Pending,
11497            Err(error) => Poll::Ready(Err(error)),
11498        }
11499    }
11500}
11501
11502/// Future returned by [`DurableOperationHandle::cancel`].
11503pub struct CancelDurableOperationCall {
11504    handle: DurableOperationHandle,
11505    emitted: bool,
11506}
11507
11508impl Future for CancelDurableOperationCall {
11509    type Output = Result<()>;
11510
11511    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11512        let ctx = self.handle.ctx.clone();
11513        let mut state = match ctx.state.lock() {
11514            Ok(state) => state,
11515            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11516        };
11517        match selection_cancellation_for_handle(&state, &self.handle) {
11518            Err(error) => return Poll::Ready(Err(error)),
11519            Ok(true) => return Poll::Ready(Ok(())),
11520            Ok(false) => {}
11521        }
11522        if recorded_selection_member_is_terminal(&state, &self.handle) {
11523            return Poll::Ready(Ok(()));
11524        }
11525        if !self.emitted {
11526            state.commands.push(json!({
11527                "type": "cancel_selection_operation",
11528                "selection_group_id": self.handle.selection_group_id,
11529                "member_key": self.handle.key,
11530                "member_index": self.handle.index,
11531                "member_base_sequence": self.handle.base_sequence,
11532                "member_size": self.handle.size,
11533                "operation_kind": self.handle.kind,
11534                "operation_identity": self.handle.identity,
11535            }));
11536            self.emitted = true;
11537        }
11538        // The cancellation command is only a request. Do not expose workflow
11539        // state after cancel until committed SelectionOperationCancelled
11540        // history proves that the request won the race with member completion.
11541        Poll::Pending
11542    }
11543}
11544
11545trait PollNestedResultExt<T> {
11546    fn flatten_result(self) -> Poll<Result<T>>;
11547}
11548
11549impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11550    fn flatten_result(self) -> Poll<Result<T>> {
11551        match self {
11552            Poll::Ready(Ok(result)) => Poll::Ready(result),
11553            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11554            Poll::Pending => Poll::Pending,
11555        }
11556    }
11557}
11558
11559struct SagaCompensation {
11560    activity_type: String,
11561    options: ActivityOptions,
11562    arguments: AvroValue,
11563    registration_order: usize,
11564}
11565
11566/// Workflow-local deterministic saga compensation helper.
11567///
11568/// Register each compensation only after its forward step succeeds. Passing
11569/// the forward `Result` to [`Saga::finish`] runs compensations sequentially in
11570/// reverse registration order after any failure, including cooperative
11571/// cancellation. Each compensation is an ordinary durable activity, so replay,
11572/// duplicate delivery, and worker restart use existing history semantics.
11573pub struct Saga {
11574    ctx: WorkflowContext,
11575    compensations: Vec<SagaCompensation>,
11576}
11577
11578impl Saga {
11579    fn new(ctx: WorkflowContext) -> Self {
11580        Self {
11581            ctx,
11582            compensations: Vec::new(),
11583        }
11584    }
11585
11586    pub fn add_compensation<T: Serialize>(
11587        &mut self,
11588        activity_type: impl Into<String>,
11589        args: T,
11590    ) -> Result<&mut Self> {
11591        self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11592    }
11593
11594    pub fn add_compensation_with_options<T: Serialize>(
11595        &mut self,
11596        activity_type: impl Into<String>,
11597        options: ActivityOptions,
11598        args: T,
11599    ) -> Result<&mut Self> {
11600        let activity_type = activity_type.into();
11601        if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11602            return Err(Error::Codec(
11603                "saga compensation activity type must be non-empty without surrounding whitespace"
11604                    .to_string(),
11605            ));
11606        }
11607        options.validate().map_err(Error::InvalidActivityOptions)?;
11608        let arguments = AvroValue::from_serialize(&args)?;
11609        let registration_order = self.compensations.len() + 1;
11610        self.compensations.push(SagaCompensation {
11611            activity_type,
11612            options,
11613            arguments,
11614            registration_order,
11615        });
11616        Ok(self)
11617    }
11618
11619    /// Compensate `initiating_failure` and return the failure that must remain.
11620    pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11621        while let Some(compensation) = self.compensations.pop() {
11622            if let Err(compensation_failure) = self
11623                .ctx
11624                .activity_with_options(
11625                    compensation.activity_type.clone(),
11626                    compensation.options,
11627                    compensation.arguments,
11628                )
11629                .await
11630            {
11631                if workflow_task_integrity_error(&compensation_failure) {
11632                    return compensation_failure;
11633                }
11634                return Error::SagaCompensationFailed(SagaCompensationFailure {
11635                    initiating_failure: Box::new(initiating_failure),
11636                    compensation_failure: Box::new(compensation_failure),
11637                    compensation_activity_type: compensation.activity_type,
11638                    compensation_registration_order: compensation.registration_order,
11639                });
11640            }
11641        }
11642        initiating_failure
11643    }
11644
11645    /// Return a successful forward value or compensate and preserve its failure.
11646    pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11647        match outcome {
11648            Ok(value) => Ok(value),
11649            Err(error) => Err(self.compensate(error).await),
11650        }
11651    }
11652}
11653
11654pub struct ActivityCall {
11655    ctx: WorkflowContext,
11656    activity_type: String,
11657    options: ActivityOptions,
11658    args: Option<Result<AvroValue>>,
11659    scheduled: bool,
11660    parallel_group_path: Vec<ParallelGroupMetadata>,
11661}
11662
11663impl ActivityCall {
11664    fn poll_avro_value(
11665        mut self: Pin<&mut Self>,
11666        _cx: &mut TaskContext<'_>,
11667    ) -> Poll<Result<AvroValue>> {
11668        let ctx = self.ctx.clone();
11669        let mut state = match ctx.state.lock() {
11670            Ok(state) => state,
11671            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11672        };
11673
11674        if self.scheduled {
11675            return Poll::Pending;
11676        }
11677
11678        let options = match self.options.validate() {
11679            Ok(options) => options,
11680            Err(error) => {
11681                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11682            }
11683        };
11684        let task_queue = options
11685            .task_queue
11686            .clone()
11687            .unwrap_or_else(|| state.task_queue.clone());
11688        let current_recorded_options = RecordedActivityOptions {
11689            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11690            // Rust schedules ordinary durable activities. The server records a
11691            // non-null mode only for a specialized execution primitive.
11692            execution_mode: RecordedSnapshotValue::Known(None),
11693            retry_policy: current_activity_retry_snapshot(&options),
11694        };
11695
11696        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11697            let sequence = recorded.sequence();
11698            match recorded {
11699                RecordedCommand::Activity {
11700                    activity_type,
11701                    options: recorded_options,
11702                    outcome,
11703                    parallel_group_path,
11704                    ..
11705                } => {
11706                    if let Err(error) = ensure_parallel_path_matches(
11707                        sequence,
11708                        parallel_group_path.as_deref(),
11709                        &self.parallel_group_path,
11710                    ) {
11711                        return Poll::Ready(Err(error));
11712                    }
11713                    if let Some(recorded_type) = activity_type {
11714                        if recorded_type != self.activity_type {
11715                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11716                                ReplayFailure::new(
11717                                    "recorded_command_detail_mismatch",
11718                                    Some(sequence),
11719                                    Some(format!("activity:{recorded_type}")),
11720                                    Some(format!("activity:{}", self.activity_type)),
11721                                    "recorded activity type differs from the current workflow command",
11722                                ),
11723                            )));
11724                        }
11725                    }
11726                    if let Some(recorded_options) = recorded_options {
11727                        if !recorded_options
11728                            .task_queue
11729                            .matches_current(&current_recorded_options.task_queue)
11730                        {
11731                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11732                                ReplayFailure::new(
11733                                    "activity_task_queue_mismatch",
11734                                    Some(sequence),
11735                                    Some(activity_options_description(&recorded_options)),
11736                                    Some(activity_options_description(&current_recorded_options)),
11737                                    "recorded activity task queue differs from the current workflow command",
11738                                ),
11739                            )));
11740                        }
11741                        if !recorded_options
11742                            .execution_mode
11743                            .matches_current(&current_recorded_options.execution_mode)
11744                        {
11745                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11746                                ReplayFailure::new(
11747                                    "activity_execution_mode_mismatch",
11748                                    Some(sequence),
11749                                    Some(activity_options_description(&recorded_options)),
11750                                    Some(activity_options_description(&current_recorded_options)),
11751                                    "recorded activity execution mode differs from the current workflow command",
11752                                ),
11753                            )));
11754                        }
11755                        if !recorded_options
11756                            .retry_policy
11757                            .matches_current(&current_recorded_options.retry_policy)
11758                        {
11759                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11760                                ReplayFailure::new(
11761                                    "activity_retry_policy_mismatch",
11762                                    Some(sequence),
11763                                    Some(activity_options_description(&recorded_options)),
11764                                    Some(activity_options_description(&current_recorded_options)),
11765                                    "recorded activity retry policy differs from the current workflow command",
11766                                ),
11767                            )));
11768                        }
11769                    }
11770                    state.command_cursor += 1;
11771                    if let Some(outcome) = outcome {
11772                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11773                    }
11774                    state.matched_recorded_pending = true;
11775                    self.scheduled = true;
11776                    return Poll::Pending;
11777                }
11778                other => {
11779                    return Poll::Ready(Err(command_mismatch(
11780                        &other,
11781                        format!("activity:{}", self.activity_type),
11782                    )));
11783                }
11784            }
11785        }
11786
11787        if !self.scheduled {
11788            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11789                Ok(args) => args,
11790                Err(error) => return Poll::Ready(Err(error)),
11791            };
11792            let arguments = normalize_avro_arguments(args);
11793            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11794                Ok(envelope) => envelope,
11795                Err(error) => return Poll::Ready(Err(error)),
11796            };
11797
11798            let mut command = serde_json::Map::from_iter([
11799                ("type".to_string(), json!("schedule_activity")),
11800                (
11801                    "activity_type".to_string(),
11802                    json!(self.activity_type.clone()),
11803                ),
11804                ("queue".to_string(), json!(task_queue)),
11805                ("arguments".to_string(), envelope),
11806            ]);
11807            for (field, value) in [
11808                ("start_to_close_timeout", options.start_to_close_timeout),
11809                (
11810                    "schedule_to_start_timeout",
11811                    options.schedule_to_start_timeout,
11812                ),
11813                (
11814                    "schedule_to_close_timeout",
11815                    options.schedule_to_close_timeout,
11816                ),
11817                ("heartbeat_timeout", options.heartbeat_timeout),
11818            ] {
11819                if let Some(value) = value {
11820                    command.insert(field.to_string(), json!(value));
11821                }
11822            }
11823            if let Some(retry_policy) = options.retry_policy {
11824                command.insert("retry_policy".to_string(), retry_policy);
11825            }
11826            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11827            state.commands.push(Value::Object(command));
11828            self.scheduled = true;
11829        }
11830
11831        Poll::Pending
11832    }
11833}
11834
11835impl Future for ActivityCall {
11836    type Output = Result<Value>;
11837
11838    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11839        match self.poll_avro_value(cx) {
11840            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11841            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11842            Poll::Pending => Poll::Pending,
11843        }
11844    }
11845}
11846
11847/// Future returned by [`WorkflowContext::sleep`].
11848pub struct TimerCall {
11849    ctx: WorkflowContext,
11850    delay_seconds: Option<u64>,
11851    scheduled: bool,
11852    matched_pending: bool,
11853    parallel_group_path: Vec<ParallelGroupMetadata>,
11854}
11855
11856impl Future for TimerCall {
11857    type Output = Result<()>;
11858
11859    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11860        if self.matched_pending {
11861            return Poll::Pending;
11862        }
11863
11864        let ctx = self.ctx.clone();
11865        let Some(requested_delay) = self.delay_seconds else {
11866            return Poll::Ready(Err(Error::TimerDurationOverflow));
11867        };
11868        let mut state = match ctx.state.lock() {
11869            Ok(state) => state,
11870            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11871        };
11872
11873        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11874            match recorded {
11875                RecordedCommand::Timer {
11876                    sequence,
11877                    delay_seconds,
11878                    fired,
11879                    parallel_group_path,
11880                    ..
11881                } => {
11882                    if let Err(error) = ensure_parallel_path_matches(
11883                        sequence,
11884                        parallel_group_path.as_deref(),
11885                        &self.parallel_group_path,
11886                    ) {
11887                        return Poll::Ready(Err(error));
11888                    }
11889                    if delay_seconds != requested_delay {
11890                        return Poll::Ready(Err(Error::NonDeterministicReplay(
11891                            ReplayFailure::new(
11892                                "timer_delay_mismatch",
11893                                Some(sequence),
11894                                Some(format!("timer:{delay_seconds}s")),
11895                                Some(format!("timer:{requested_delay}s")),
11896                                "recorded timer delay differs from the current workflow command",
11897                            ),
11898                        )));
11899                    }
11900                    state.command_cursor += 1;
11901                    if fired {
11902                        return Poll::Ready(Ok(()));
11903                    }
11904                    state.matched_recorded_pending = true;
11905                    self.scheduled = true;
11906                    self.matched_pending = true;
11907                    return Poll::Pending;
11908                }
11909                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11910            }
11911        }
11912
11913        if !self.scheduled {
11914            let mut command = serde_json::Map::from_iter([
11915                ("type".to_string(), json!("start_timer")),
11916                ("delay_seconds".to_string(), json!(requested_delay)),
11917            ]);
11918            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11919            state.commands.push(Value::Object(command));
11920            self.scheduled = true;
11921        }
11922
11923        Poll::Pending
11924    }
11925}
11926
11927/// Future returned by [`WorkflowContext::wait_condition`].
11928pub struct ConditionWaitCall {
11929    ctx: WorkflowContext,
11930    options: ConditionWaitOptions,
11931    predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11932    occurrence_id: Option<String>,
11933    opened_wait: bool,
11934    parallel_group_path: Vec<ParallelGroupMetadata>,
11935}
11936
11937impl Future for ConditionWaitCall {
11938    type Output = Result<ConditionWaitResult>;
11939
11940    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11941        if self.opened_wait {
11942            return Poll::Pending;
11943        }
11944
11945        let options = match self.options.validate() {
11946            Ok(options) => options,
11947            Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11948        };
11949        let ctx = self.ctx.clone();
11950        let occurrence_id = match self.occurrence_id.as_ref() {
11951            Some(occurrence_id) => occurrence_id.clone(),
11952            None => {
11953                let mut state = match ctx.state.lock() {
11954                    Ok(state) => state,
11955                    Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11956                };
11957                let ordinal = state.condition_wait_occurrence_counter;
11958                state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11959                    Some(next) => next,
11960                    None => {
11961                        return Poll::Ready(Err(Error::WorkerLoop(
11962                            "condition wait occurrence counter overflowed".to_string(),
11963                        )))
11964                    }
11965                };
11966                let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11967                drop(state);
11968                self.occurrence_id = Some(occurrence_id.clone());
11969                occurrence_id
11970            }
11971        };
11972
11973        let recorded_result = {
11974            let mut state = match ctx.state.lock() {
11975                Ok(state) => state,
11976                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11977            };
11978            let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11979                drop(state);
11980                return self.poll_new_condition(options);
11981            };
11982            if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11983                return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11984            }
11985
11986            let mut cursor = state.command_cursor;
11987            let mut result = None;
11988            loop {
11989                let Some(RecordedCommand::ConditionWait {
11990                    sequence,
11991                    occurrence_id: recorded_occurrence_id,
11992                    condition_key,
11993                    predicate_identity,
11994                    timeout_seconds,
11995                    result: recorded_result,
11996                    parallel_group_path,
11997                    ..
11998                }) = state.recorded_commands.get(cursor)
11999                else {
12000                    break;
12001                };
12002
12003                if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
12004                    break;
12005                }
12006                if let Err(error) = ensure_parallel_path_matches(
12007                    *sequence,
12008                    parallel_group_path.as_deref(),
12009                    &self.parallel_group_path,
12010                ) {
12011                    return Poll::Ready(Err(error));
12012                }
12013                if let Err(error) = validate_recorded_condition_wait(
12014                    *sequence,
12015                    recorded_occurrence_id,
12016                    condition_key.as_deref(),
12017                    predicate_identity,
12018                    *timeout_seconds,
12019                    &occurrence_id,
12020                    &options,
12021                ) {
12022                    return Poll::Ready(Err(error));
12023                }
12024                if result == Some(ConditionWaitResult::TimedOut) {
12025                    return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
12026                        "condition_wait_reopened_after_timeout",
12027                        Some(*sequence),
12028                        Some("timed-out condition is terminal".to_string()),
12029                        Some("another physical wait-open".to_string()),
12030                        "condition history reopened one logical wait after its durable timeout",
12031                    ))));
12032                }
12033                result = *recorded_result;
12034                cursor += 1;
12035            }
12036            state.command_cursor = cursor;
12037            result
12038        };
12039
12040        if let Some(result) = recorded_result {
12041            return Poll::Ready(Ok(result));
12042        }
12043
12044        self.poll_open_condition(options)
12045    }
12046}
12047
12048impl ConditionWaitCall {
12049    fn poll_new_condition(
12050        self: Pin<&mut Self>,
12051        options: ValidatedConditionWaitOptions,
12052    ) -> Poll<Result<ConditionWaitResult>> {
12053        self.poll_open_condition(options)
12054    }
12055
12056    fn poll_open_condition(
12057        mut self: Pin<&mut Self>,
12058        options: ValidatedConditionWaitOptions,
12059    ) -> Poll<Result<ConditionWaitResult>> {
12060        let selection_member = self
12061            .parallel_group_path
12062            .first()
12063            .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
12064        match (self.predicate)() {
12065            Ok(true) if !selection_member => {
12066                return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
12067            }
12068            Ok(_) => {}
12069            Err(error) => return Poll::Ready(Err(error)),
12070        }
12071        if options.timeout_seconds == Some(0) && !selection_member {
12072            return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
12073        }
12074
12075        let ctx = self.ctx.clone();
12076        let mut state = match ctx.state.lock() {
12077            Ok(state) => state,
12078            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12079        };
12080        let mut command = serde_json::Map::from_iter([
12081            ("type".to_string(), json!("open_condition_wait")),
12082            (
12083                "condition_wait_occurrence_id".to_string(),
12084                json!(self.occurrence_id.as_deref().unwrap_or_default()),
12085            ),
12086            ("condition_key".to_string(), json!(options.condition_key)),
12087            (
12088                "condition_definition_fingerprint".to_string(),
12089                json!(options.predicate_identity),
12090            ),
12091        ]);
12092        if let Some(timeout_seconds) = options.timeout_seconds {
12093            command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
12094        }
12095        apply_parallel_group_path(&mut command, &self.parallel_group_path);
12096        state.commands.push(Value::Object(command));
12097        drop(state);
12098        self.opened_wait = true;
12099        Poll::Pending
12100    }
12101}
12102
12103fn validate_recorded_condition_wait(
12104    sequence: u64,
12105    recorded_occurrence_id: &str,
12106    recorded_key: Option<&str>,
12107    recorded_predicate_identity: &str,
12108    recorded_timeout_seconds: Option<u64>,
12109    current_occurrence_id: &str,
12110    current: &ValidatedConditionWaitOptions,
12111) -> Result<()> {
12112    if recorded_occurrence_id != current_occurrence_id {
12113        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12114            "condition_wait_occurrence_mismatch",
12115            Some(sequence),
12116            Some(recorded_occurrence_id.to_string()),
12117            Some(current_occurrence_id.to_string()),
12118            "recorded condition occurrence differs from the current authored wait position",
12119        )));
12120    }
12121    if recorded_key != Some(current.condition_key.as_str()) {
12122        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12123            "condition_wait_key_mismatch",
12124            Some(sequence),
12125            recorded_key.map(str::to_string),
12126            Some(current.condition_key.clone()),
12127            "recorded condition identity differs from the current workflow wait",
12128        )));
12129    }
12130    if recorded_predicate_identity != current.predicate_identity {
12131        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12132            "condition_wait_predicate_mismatch",
12133            Some(sequence),
12134            Some(recorded_predicate_identity.to_string()),
12135            Some(current.predicate_identity.clone()),
12136            "recorded condition predicate behavior differs from current workflow code",
12137        )));
12138    }
12139    if recorded_timeout_seconds != current.timeout_seconds {
12140        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12141            "condition_wait_timeout_mismatch",
12142            Some(sequence),
12143            recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12144            current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12145            "recorded condition timeout differs from the current workflow wait",
12146        )));
12147    }
12148    Ok(())
12149}
12150
12151/// Future returned by [`WorkflowContext::start_child_workflow`].
12152pub struct ChildWorkflowCall {
12153    ctx: WorkflowContext,
12154    workflow_type: String,
12155    options: ChildWorkflowOptions,
12156    args: Option<Result<AvroValue>>,
12157    scheduled: bool,
12158    matched_pending: bool,
12159    parallel_group_path: Vec<ParallelGroupMetadata>,
12160}
12161
12162impl ChildWorkflowCall {
12163    fn poll_avro_value(
12164        mut self: Pin<&mut Self>,
12165        _cx: &mut TaskContext<'_>,
12166    ) -> Poll<Result<ChildWorkflowAvroResult>> {
12167        if self.matched_pending {
12168            return Poll::Pending;
12169        }
12170
12171        let ctx = self.ctx.clone();
12172        let mut state = match ctx.state.lock() {
12173            Ok(state) => state,
12174            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12175        };
12176
12177        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12178            let sequence = recorded.sequence();
12179            match recorded {
12180                RecordedCommand::ChildWorkflow {
12181                    workflow_type,
12182                    outcome,
12183                    parallel_group_path,
12184                    ..
12185                } => {
12186                    if let Err(error) = ensure_parallel_path_matches(
12187                        sequence,
12188                        parallel_group_path.as_deref(),
12189                        &self.parallel_group_path,
12190                    ) {
12191                        return Poll::Ready(Err(error));
12192                    }
12193                    if let Some(recorded_type) = workflow_type {
12194                        if recorded_type != self.workflow_type {
12195                            return Poll::Ready(Err(Error::NonDeterministicReplay(
12196                                ReplayFailure::new(
12197                                    "recorded_command_detail_mismatch",
12198                                    Some(sequence),
12199                                    Some(format!("child workflow:{recorded_type}")),
12200                                    Some(format!("child workflow:{}", self.workflow_type)),
12201                                    "recorded child workflow type differs from the current workflow command",
12202                                ),
12203                            )));
12204                        }
12205                    }
12206                    state.command_cursor += 1;
12207                    if let Some(outcome) = outcome {
12208                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12209                    }
12210                    state.matched_recorded_pending = true;
12211                    self.scheduled = true;
12212                    self.matched_pending = true;
12213                    return Poll::Pending;
12214                }
12215                other => {
12216                    return Poll::Ready(Err(command_mismatch(
12217                        &other,
12218                        format!("child workflow:{}", self.workflow_type),
12219                    )));
12220                }
12221            }
12222        }
12223
12224        if !self.scheduled {
12225            if self.options.task_queue.trim().is_empty() {
12226                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12227                    "task_queue must not be empty".to_string(),
12228                )));
12229            }
12230            for (name, value) in [
12231                (
12232                    "execution_timeout_seconds",
12233                    self.options.execution_timeout_seconds,
12234                ),
12235                ("run_timeout_seconds", self.options.run_timeout_seconds),
12236            ] {
12237                if value == Some(0) {
12238                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12239                        "{name} must be at least 1"
12240                    ))));
12241                }
12242            }
12243
12244            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12245                Ok(args) => args,
12246                Err(error) => return Poll::Ready(Err(error)),
12247            };
12248            let arguments = match encode_typed_envelope(
12249                &normalize_avro_arguments(args),
12250                &state.payload_codec,
12251            ) {
12252                Ok(arguments) => arguments,
12253                Err(error) => return Poll::Ready(Err(error)),
12254            };
12255            let mut command = json!({
12256                "type": "start_child_workflow",
12257                "workflow_type": self.workflow_type,
12258                "queue": self.options.task_queue,
12259                "parent_close_policy": self.options.parent_close_policy.as_str(),
12260                "arguments": arguments,
12261            });
12262            let object = command
12263                .as_object_mut()
12264                .expect("child workflow command is always an object");
12265            if let Some(policy) = &self.options.retry_policy {
12266                let mut retry_policy = serde_json::Map::new();
12267                if let Some(max_attempts) = policy.max_attempts {
12268                    if max_attempts == 0 {
12269                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12270                            "retry_policy.max_attempts must be at least 1".to_string(),
12271                        )));
12272                    }
12273                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12274                }
12275                if !policy.backoff_seconds.is_empty() {
12276                    retry_policy
12277                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12278                }
12279                if !policy.non_retryable_error_types.is_empty() {
12280                    retry_policy.insert(
12281                        "non_retryable_error_types".to_string(),
12282                        json!(policy.non_retryable_error_types),
12283                    );
12284                }
12285                if retry_policy.is_empty() {
12286                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12287                        "retry_policy must configure at least one field".to_string(),
12288                    )));
12289                }
12290                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12291            }
12292            if let Some(seconds) = self.options.execution_timeout_seconds {
12293                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12294            }
12295            if let Some(seconds) = self.options.run_timeout_seconds {
12296                object.insert("run_timeout_seconds".to_string(), json!(seconds));
12297            }
12298            apply_parallel_group_path(object, &self.parallel_group_path);
12299            state.commands.push(command);
12300            self.scheduled = true;
12301        }
12302
12303        Poll::Pending
12304    }
12305}
12306
12307impl Future for ChildWorkflowCall {
12308    type Output = Result<ChildWorkflowResult>;
12309
12310    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12311        match self.poll_avro_value(cx) {
12312            Poll::Ready(Ok(result)) => match result.result.into_json() {
12313                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12314                    parent: result.parent,
12315                    child: result.child,
12316                    child_workflow_type: result.child_workflow_type,
12317                    result: projected,
12318                })),
12319                Err(error) => Poll::Ready(Err(error)),
12320            },
12321            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12322            Poll::Pending => Poll::Pending,
12323        }
12324    }
12325}
12326
12327fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12328    Error::NonDeterministicReplay(ReplayFailure::new(
12329        "recorded_command_mismatch",
12330        Some(recorded.sequence()),
12331        Some(recorded.shape().to_string()),
12332        Some(actual.into()),
12333        "current workflow command does not match the recorded durable command sequence",
12334    ))
12335}
12336
12337pub struct SignalCall {
12338    ctx: WorkflowContext,
12339    signal_name: String,
12340    runtime_reserved_allowed: bool,
12341    opened_wait: bool,
12342    matched_pending: bool,
12343    parallel_group_path: Vec<ParallelGroupMetadata>,
12344}
12345
12346impl SignalCall {
12347    fn poll_avro_value(
12348        mut self: Pin<&mut Self>,
12349        _cx: &mut TaskContext<'_>,
12350    ) -> Poll<Result<Vec<AvroValue>>> {
12351        if self.matched_pending {
12352            return Poll::Pending;
12353        }
12354        if !self.runtime_reserved_allowed {
12355            if let Err(error) = validate_user_signal_name(&self.signal_name) {
12356                return Poll::Ready(Err(error));
12357            }
12358        }
12359
12360        let ctx = self.ctx.clone();
12361        let mut state = match ctx.state.lock() {
12362            Ok(state) => state,
12363            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12364        };
12365
12366        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12367            match recorded {
12368                RecordedCommand::SignalWait {
12369                    sequence,
12370                    signal_name,
12371                    value,
12372                    parallel_group_path,
12373                } => {
12374                    if let Err(error) = ensure_parallel_path_matches(
12375                        sequence,
12376                        parallel_group_path.as_deref(),
12377                        &self.parallel_group_path,
12378                    ) {
12379                        return Poll::Ready(Err(error));
12380                    }
12381                    if signal_name != self.signal_name {
12382                        return Poll::Ready(Err(Error::NonDeterministicReplay(
12383                            ReplayFailure::new(
12384                                "recorded_command_detail_mismatch",
12385                                Some(sequence),
12386                                Some(format!("signal wait:{signal_name}")),
12387                                Some(format!("signal wait:{}", self.signal_name)),
12388                                "recorded signal name differs from the current workflow command",
12389                            ),
12390                        )));
12391                    }
12392
12393                    state.command_cursor += 1;
12394                    if let Some(value) = value {
12395                        return Poll::Ready(Ok(value));
12396                    }
12397                    if state
12398                        .resume_signal
12399                        .as_ref()
12400                        .is_some_and(|signal| signal.signal_name == self.signal_name)
12401                    {
12402                        let signal = state
12403                            .resume_signal
12404                            .take()
12405                            .expect("matching resume signal is present");
12406                        return Poll::Ready(Ok(signal.arguments));
12407                    }
12408
12409                    state.matched_recorded_pending = true;
12410                    self.opened_wait = true;
12411                    self.matched_pending = true;
12412                    return Poll::Pending;
12413                }
12414                other => {
12415                    return Poll::Ready(Err(command_mismatch(
12416                        &other,
12417                        format!("signal wait:{}", self.signal_name),
12418                    )));
12419                }
12420            }
12421        }
12422
12423        if state
12424            .resume_signal
12425            .as_ref()
12426            .is_some_and(|signal| signal.signal_name == self.signal_name)
12427        {
12428            let signal = state
12429                .resume_signal
12430                .take()
12431                .expect("matching resume signal is present");
12432            return Poll::Ready(Ok(signal.arguments));
12433        }
12434
12435        if !self.opened_wait {
12436            let mut command = serde_json::Map::from_iter([
12437                ("type".to_string(), json!("open_signal_wait")),
12438                ("signal_name".to_string(), json!(self.signal_name)),
12439            ]);
12440            apply_parallel_group_path(&mut command, &self.parallel_group_path);
12441            state.commands.push(Value::Object(command));
12442            self.opened_wait = true;
12443        }
12444
12445        Poll::Pending
12446    }
12447}
12448
12449impl Future for SignalCall {
12450    type Output = Result<Vec<Value>>;
12451
12452    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12453        match self.poll_avro_value(cx) {
12454            Poll::Ready(Ok(values)) => Poll::Ready(
12455                values
12456                    .into_iter()
12457                    .map(AvroValue::into_json)
12458                    .collect::<Result<Vec<_>>>(),
12459            ),
12460            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12461            Poll::Pending => Poll::Pending,
12462        }
12463    }
12464}
12465
12466#[derive(Clone, Debug)]
12467pub struct ActivityContext {
12468    client: Client,
12469    pub task_id: String,
12470    pub activity_attempt_id: String,
12471    pub lease_owner: String,
12472    pub activity_type: String,
12473    pub attempt_number: u64,
12474    pub task_queue: String,
12475    pub worker_id: String,
12476}
12477
12478impl ActivityContext {
12479    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12480        self.client
12481            .heartbeat_activity_task(
12482                &self.task_id,
12483                &self.activity_attempt_id,
12484                &self.lease_owner,
12485                details,
12486            )
12487            .await
12488    }
12489}
12490
12491fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12492    validate_payload_codec(codec)?;
12493    match value {
12494        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12495            value, codec,
12496        )?)),
12497        None => Ok(AvroValue::Array(Vec::new())),
12498    }
12499}
12500
12501fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12502    let Some(signal_name) = task
12503        .signal_name
12504        .as_deref()
12505        .filter(|value| !value.is_empty())
12506    else {
12507        return Ok(None);
12508    };
12509    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12510    let AvroValue::Array(arguments) = decoded else {
12511        unreachable!("normalize_avro_arguments always returns an array");
12512    };
12513
12514    Ok(Some(ResumeSignal {
12515        signal_name: signal_name.to_string(),
12516        arguments,
12517    }))
12518}
12519
12520fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12521    validate_payload_codec(&task.payload_codec)?;
12522    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12523    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12524    for event in &task.history_events {
12525        validate_history_event_payloads(event, &task.payload_codec)?;
12526    }
12527    Ok(())
12528}
12529
12530fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12531    validate_payload_codec(&task.payload_codec)?;
12532    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12533}
12534
12535fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12536    validate_payload_codec(&task.payload_codec)?;
12537    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12538    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12539    for event in &task.history_events {
12540        validate_history_event_payloads(event, &task.payload_codec)?;
12541    }
12542
12543    let Some(export) = task.history_export.as_ref() else {
12544        return Ok(());
12545    };
12546    let export_codec = match export.get("payloads") {
12547        Some(payloads) => declared_payload_codec(payloads, "codec")?,
12548        None => None,
12549    }
12550    .unwrap_or(&task.payload_codec);
12551    validate_payload_codec(export_codec)?;
12552
12553    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12554        for event in events {
12555            let event_type = event
12556                .get("event_type")
12557                .or_else(|| event.get("type"))
12558                .and_then(Value::as_str)
12559                .unwrap_or_default();
12560            if let Some(payload) = event.get("payload") {
12561                validate_history_payloads(event_type, payload, export_codec)?;
12562            }
12563        }
12564    }
12565    for signal in export
12566        .get("signals")
12567        .and_then(Value::as_array)
12568        .into_iter()
12569        .flatten()
12570    {
12571        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12572        validate_payload_codec(codec)?;
12573        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12574    }
12575    for activity in export
12576        .get("activities")
12577        .and_then(Value::as_array)
12578        .into_iter()
12579        .flatten()
12580    {
12581        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12582        validate_payload_codec(codec)?;
12583        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12584        validate_optional_inbound_payload(activity.get("result"), codec)?;
12585    }
12586    Ok(())
12587}
12588
12589fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12590    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12591}
12592
12593fn validate_history_payloads(
12594    event_type: &str,
12595    payload: &Value,
12596    fallback_codec: &str,
12597) -> Result<()> {
12598    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12599    validate_payload_codec(codec)?;
12600    for field in history_payload_fields(event_type) {
12601        validate_optional_inbound_payload(payload.get(*field), codec)?;
12602    }
12603    Ok(())
12604}
12605
12606const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12607
12608fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12609    match event_type {
12610        "ActivityCompleted" => &["result"],
12611        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12612        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12613        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12614        "ChildRunCompleted" => &["result", "output"],
12615        "WorkflowCompleted" => &["output"],
12616        "ServiceCallStarted"
12617        | "ServiceCallCompleted"
12618        | "ServiceCallFailed"
12619        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12620        _ => &[],
12621    }
12622}
12623
12624fn signal_history_payload(payload: &Value) -> Option<&Value> {
12625    SIGNAL_HISTORY_PAYLOAD_FIELDS
12626        .iter()
12627        .find_map(|field| payload.get(*field))
12628}
12629
12630fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12631    match value.get(field) {
12632        None => Ok(None),
12633        Some(Value::String(codec)) => Ok(Some(codec)),
12634        Some(_) => Err(invalid_payload_envelope()),
12635    }
12636}
12637
12638fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12639    validate_payload_codec(codec)?;
12640    if let Some(value) = value.filter(|value| !value.is_null()) {
12641        decode_wire_avro_value(value, codec)?;
12642    }
12643    Ok(())
12644}
12645
12646fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12647    let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12648        invalid_recorded_history(
12649            "parallel_group_metadata_invalid",
12650            sequence,
12651            "non-empty parallel_group_id",
12652            &payload.to_string(),
12653            "parallel-group history is missing its stable identity",
12654        )
12655    })?;
12656    let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12657        invalid_recorded_history(
12658            "parallel_group_metadata_invalid",
12659            sequence,
12660            "activity, child, timer, signal, condition, or mixed group kind",
12661            &payload.to_string(),
12662            "parallel-group history is missing its group kind",
12663        )
12664    })?;
12665    if !matches!(
12666        kind.as_str(),
12667        "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12668    ) {
12669        return Err(invalid_recorded_history(
12670            "parallel_group_metadata_invalid",
12671            sequence,
12672            "activity, child, timer, signal, condition, or mixed group kind",
12673            &kind,
12674            "parallel-group history contains an unsupported group kind",
12675        ));
12676    }
12677    let base_sequence = payload
12678        .get("parallel_group_base_sequence")
12679        .and_then(value_as_u64)
12680        .filter(|value| *value > 0)
12681        .ok_or_else(|| {
12682            invalid_recorded_history(
12683                "parallel_group_metadata_invalid",
12684                sequence,
12685                "positive parallel_group_base_sequence",
12686                &payload.to_string(),
12687                "parallel-group history contains an invalid base sequence",
12688            )
12689        })?;
12690    let size = payload
12691        .get("parallel_group_size")
12692        .and_then(value_as_u64)
12693        .and_then(|value| usize::try_from(value).ok())
12694        .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12695        .ok_or_else(|| {
12696            invalid_recorded_history(
12697                "parallel_group_metadata_invalid",
12698                sequence,
12699                "bounded positive parallel_group_size",
12700                &payload.to_string(),
12701                "parallel-group history contains an invalid group size",
12702            )
12703        })?;
12704    let index = payload
12705        .get("parallel_group_index")
12706        .and_then(value_as_u64)
12707        .and_then(|value| usize::try_from(value).ok())
12708        .filter(|value| *value < size)
12709        .ok_or_else(|| {
12710            invalid_recorded_history(
12711                "parallel_group_metadata_invalid",
12712                sequence,
12713                "parallel_group_index within group bounds",
12714                &payload.to_string(),
12715                "parallel-group history contains an invalid member index",
12716            )
12717        })?;
12718    if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12719        return Err(invalid_recorded_history(
12720            "parallel_group_metadata_invalid",
12721            sequence,
12722            "base sequence plus member index equals workflow sequence",
12723            &payload.to_string(),
12724            "parallel-group path does not preserve durable workflow position",
12725        ));
12726    }
12727    let mode = payload
12728        .get("parallel_group_mode")
12729        .and_then(Value::as_str)
12730        .unwrap_or("all");
12731    if !matches!(mode, "all" | "select") {
12732        return Err(invalid_recorded_history(
12733            "parallel_group_metadata_invalid",
12734            sequence,
12735            "parallel group mode all or select",
12736            mode,
12737            "parallel-group history contains an unsupported group mode",
12738        ));
12739    }
12740    let expected_id = if mode == "select" {
12741        format!("select-calls:{base_sequence}:{size}")
12742    } else {
12743        format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12744    };
12745    if group_id != expected_id {
12746        return Err(invalid_recorded_history(
12747            "parallel_group_metadata_invalid",
12748            sequence,
12749            &expected_id,
12750            &group_id,
12751            "parallel-group history contains an incompatible stable group ID",
12752        ));
12753    }
12754    let selection_member_key = if mode == "select" {
12755        Some(selection_key_from_value(
12756            payload.get("selection_member_key"),
12757            sequence,
12758        )?)
12759    } else {
12760        None
12761    };
12762    let selection_member_index = if mode == "select" {
12763        Some(required_parallel_usize(
12764            payload,
12765            "selection_member_index",
12766            sequence,
12767        )?)
12768    } else {
12769        None
12770    };
12771    let selection_member_base_sequence = if mode == "select" {
12772        Some(
12773            payload
12774                .get("selection_member_base_sequence")
12775                .and_then(value_as_u64)
12776                .filter(|value| *value >= base_sequence)
12777                .ok_or_else(|| {
12778                    invalid_recorded_history(
12779                        "parallel_group_metadata_invalid",
12780                        sequence,
12781                        "selection member base within its group",
12782                        &payload.to_string(),
12783                        "selection history contains an invalid member base sequence",
12784                    )
12785                })?,
12786        )
12787    } else {
12788        None
12789    };
12790    let selection_member_size = if mode == "select" {
12791        let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12792        if member_size == 0 {
12793            return Err(invalid_recorded_history(
12794                "parallel_group_metadata_invalid",
12795                sequence,
12796                "positive selection member size",
12797                &payload.to_string(),
12798                "selection history contains an invalid member size",
12799            ));
12800        }
12801        Some(member_size)
12802    } else {
12803        None
12804    };
12805    let selection_member_kind = if mode == "select" {
12806        let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12807            invalid_recorded_history(
12808                "parallel_group_metadata_invalid",
12809                sequence,
12810                "selection member operation kind",
12811                &payload.to_string(),
12812                "selection history is missing its authored member kind",
12813            )
12814        })?;
12815        if !matches!(
12816            kind.as_str(),
12817            "activity" | "child" | "timer" | "signal" | "condition" | "group"
12818        ) {
12819            return Err(invalid_recorded_history(
12820                "parallel_group_metadata_invalid",
12821                sequence,
12822                "activity, child, timer, signal, condition, or group selection member kind",
12823                &kind,
12824                "selection history contains an unsupported member kind",
12825            ));
12826        }
12827        Some(kind)
12828    } else {
12829        None
12830    };
12831    if let (Some(member_base), Some(member_size)) =
12832        (selection_member_base_sequence, selection_member_size)
12833    {
12834        let member_end = member_base
12835            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12836            .ok_or_else(|| {
12837                invalid_recorded_history(
12838                    "parallel_group_metadata_invalid",
12839                    sequence,
12840                    "bounded selection member range",
12841                    &payload.to_string(),
12842                    "selection member range overflowed",
12843                )
12844            })?;
12845        let group_end = base_sequence
12846            .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12847            .unwrap_or(u64::MAX);
12848        if sequence < member_base || sequence >= member_end || member_end > group_end {
12849            return Err(invalid_recorded_history(
12850                "parallel_group_metadata_invalid",
12851                sequence,
12852                "workflow sequence within one bounded selection member",
12853                &payload.to_string(),
12854                "selection member range does not contain its durable leaf",
12855            ));
12856        }
12857    }
12858    Ok(ParallelGroupMetadata {
12859        parallel_group_id: group_id,
12860        parallel_group_kind: kind,
12861        parallel_group_base_sequence: base_sequence,
12862        parallel_group_size: size,
12863        parallel_group_index: index,
12864        parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12865        selection_member_key,
12866        selection_member_index,
12867        selection_member_base_sequence,
12868        selection_member_size,
12869        selection_member_kind,
12870    })
12871}
12872
12873fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12874    payload
12875        .get(field)
12876        .and_then(value_as_u64)
12877        .and_then(|value| usize::try_from(value).ok())
12878        .ok_or_else(|| {
12879            invalid_recorded_history(
12880                "parallel_group_metadata_invalid",
12881                sequence,
12882                &format!("non-negative integer {field}"),
12883                &payload.to_string(),
12884                "selection history contains invalid member metadata",
12885            )
12886        })
12887}
12888
12889fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12890    match value {
12891        Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12892        Some(value) => value_as_u64(value)
12893            .and_then(|value| usize::try_from(value).ok())
12894            .map(SelectionKey::Index)
12895            .ok_or_else(|| {
12896                invalid_recorded_history(
12897                    "selection_member_key_invalid",
12898                    sequence,
12899                    "non-empty string or non-negative integer member key",
12900                    &value.to_string(),
12901                    "selection history contains an invalid member key",
12902                )
12903            }),
12904        None => Err(invalid_recorded_history(
12905            "selection_member_key_missing",
12906            sequence,
12907            "selection_member_key",
12908            "<missing>",
12909            "selection history is missing its stable member key",
12910        )),
12911    }
12912}
12913
12914fn recorded_parallel_group_path(
12915    events: &[&HistoryEvent],
12916    sequence: u64,
12917) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12918    let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12919    for event in events {
12920        let payload = &event.payload;
12921        let has_metadata = payload.get("parallel_group_path").is_some()
12922            || payload.get("parallel_group_id").is_some()
12923            || payload.get("parallel_group_kind").is_some()
12924            || payload.get("parallel_group_base_sequence").is_some()
12925            || payload.get("parallel_group_size").is_some()
12926            || payload.get("parallel_group_index").is_some()
12927            || payload.get("parallel_group_mode").is_some()
12928            || payload.get("selection_member_key").is_some();
12929        if !has_metadata {
12930            continue;
12931        }
12932
12933        let top_level = recorded_parallel_group_entry(payload, sequence)?;
12934        let path = match payload.get("parallel_group_path") {
12935            None => vec![top_level.clone()],
12936            Some(Value::Array(entries)) if !entries.is_empty() => entries
12937                .iter()
12938                .map(|entry| recorded_parallel_group_entry(entry, sequence))
12939                .collect::<Result<Vec<_>>>()?,
12940            Some(value) => {
12941                return Err(invalid_recorded_history(
12942                    "parallel_group_metadata_invalid",
12943                    sequence,
12944                    "non-empty parallel_group_path list",
12945                    &value.to_string(),
12946                    "parallel-group history contains an invalid group path",
12947                ));
12948            }
12949        };
12950        if path.last() != Some(&top_level) {
12951            return Err(invalid_recorded_history(
12952                "parallel_group_metadata_invalid",
12953                sequence,
12954                &serde_json::to_string(&path.last()).unwrap_or_default(),
12955                &serde_json::to_string(&top_level).unwrap_or_default(),
12956                "parallel-group top-level fields do not match the innermost path entry",
12957            ));
12958        }
12959        if recorded.as_ref().is_some_and(|existing| existing != &path) {
12960            return Err(invalid_recorded_history(
12961                "parallel_group_history_conflict",
12962                sequence,
12963                &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12964                &serde_json::to_string(&path).unwrap_or_default(),
12965                "parallel-group metadata changed between scheduling and resolution history",
12966            ));
12967        }
12968        recorded = Some(path);
12969    }
12970    Ok(recorded)
12971}
12972
12973fn recorded_commands(
12974    events: &[HistoryEvent],
12975    fallback_codec: &str,
12976    parent: WorkflowIdentity,
12977) -> Result<Vec<RecordedCommand>> {
12978    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12979    let mut last_new_sequence = None;
12980
12981    for event in events {
12982        let is_activity = matches!(
12983            event.event_type.as_str(),
12984            "ActivityScheduled"
12985                | "ActivityStarted"
12986                | "ActivityHeartbeatRecorded"
12987                | "ActivityRetryScheduled"
12988                | "ActivityCompleted"
12989                | "ActivityFailed"
12990                | "ActivityCancelled"
12991                | "ActivityTimedOut"
12992        );
12993        let is_workflow_timer = matches!(
12994            event.event_type.as_str(),
12995            "TimerScheduled" | "TimerCancelled" | "TimerFired"
12996        ) && !is_internal_timer_event(event);
12997        let is_child_workflow = matches!(
12998            event.event_type.as_str(),
12999            "ChildWorkflowScheduled"
13000                | "ChildRunCompleted"
13001                | "ChildRunFailed"
13002                | "ChildRunCancelled"
13003                | "ChildRunTerminated"
13004        );
13005        let is_signal_wait = is_recorded_signal_wait_event(event);
13006        let is_condition_wait = is_recorded_condition_wait_event(event);
13007        let is_search_attributes = event.event_type == "SearchAttributesUpserted";
13008        let is_side_effect = event.event_type == "SideEffectRecorded";
13009        let is_version_marker = event.event_type == "VersionMarkerRecorded";
13010        let is_memo = event.event_type == "MemoUpserted";
13011        if !is_activity
13012            && !is_workflow_timer
13013            && !is_child_workflow
13014            && !is_signal_wait
13015            && !is_condition_wait
13016            && !is_search_attributes
13017            && !is_side_effect
13018            && !is_version_marker
13019            && !is_memo
13020        {
13021            continue;
13022        }
13023
13024        let sequence = durable_event_sequence(event).ok_or_else(|| {
13025            Error::NonDeterministicReplay(ReplayFailure::new(
13026                "durable_command_sequence_missing",
13027                None,
13028                Some("positive workflow sequence".to_string()),
13029                Some(event.event_type.clone()),
13030                "durable command history event has no workflow sequence",
13031            ))
13032        })?;
13033        if sequence == 0 {
13034            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
13035                "durable_command_sequence_invalid",
13036                Some(sequence),
13037                Some("positive workflow sequence".to_string()),
13038                Some(sequence.to_string()),
13039                "durable command history uses an invalid workflow sequence",
13040            )));
13041        }
13042        if !events_by_sequence.contains_key(&sequence) {
13043            if let Some(previous) = last_new_sequence {
13044                if sequence < previous {
13045                    return Err(invalid_recorded_history(
13046                        "durable_command_sequence_mismatch",
13047                        sequence,
13048                        &format!("workflow sequence greater than {previous}"),
13049                        &sequence.to_string(),
13050                        "durable commands are not strictly ordered by their recorded workflow sequence",
13051                    ));
13052                }
13053            }
13054            last_new_sequence = Some(sequence);
13055        }
13056        events_by_sequence.entry(sequence).or_default().push(event);
13057    }
13058
13059    let commands: Vec<RecordedCommand> = events_by_sequence
13060        .into_iter()
13061        .map(|(sequence, sequence_events)| {
13062            let activity_events: Vec<_> = sequence_events
13063                .iter()
13064                .copied()
13065                .filter(|event| event.event_type.starts_with("Activity"))
13066                .collect();
13067            let timer_events: Vec<_> = sequence_events
13068                .iter()
13069                .copied()
13070                .filter(|event| event.event_type.starts_with("Timer"))
13071                .collect();
13072            let child_events: Vec<_> = sequence_events
13073                .iter()
13074                .copied()
13075                .filter(|event| {
13076                    event.event_type == "ChildWorkflowScheduled"
13077                        || event.event_type.starts_with("ChildRun")
13078                })
13079                .collect();
13080            let signal_wait_events: Vec<_> = sequence_events
13081                .iter()
13082                .copied()
13083                .filter(|event| is_recorded_signal_wait_event(event))
13084                .collect();
13085            let condition_wait_events: Vec<_> = sequence_events
13086                .iter()
13087                .copied()
13088                .filter(|event| is_recorded_condition_wait_event(event))
13089                .collect();
13090            let search_attribute_events: Vec<_> = sequence_events
13091                .iter()
13092                .copied()
13093                .filter(|event| event.event_type == "SearchAttributesUpserted")
13094                .collect();
13095            let side_effect_events: Vec<_> = sequence_events
13096                .iter()
13097                .copied()
13098                .filter(|event| event.event_type == "SideEffectRecorded")
13099                .collect();
13100            let version_marker_events: Vec<_> = sequence_events
13101                .iter()
13102                .copied()
13103                .filter(|event| event.event_type == "VersionMarkerRecorded")
13104                .collect();
13105            let memo_events: Vec<_> = sequence_events
13106                .iter()
13107                .copied()
13108                .filter(|event| event.event_type == "MemoUpserted")
13109                .collect();
13110
13111            let command_kind_count = usize::from(!activity_events.is_empty())
13112                + usize::from(!timer_events.is_empty())
13113                + usize::from(!child_events.is_empty())
13114                + usize::from(!signal_wait_events.is_empty())
13115                + usize::from(!condition_wait_events.is_empty())
13116                + usize::from(!search_attribute_events.is_empty())
13117                + usize::from(!side_effect_events.is_empty())
13118                + usize::from(!version_marker_events.is_empty())
13119                + usize::from(!memo_events.is_empty());
13120            if command_kind_count > 1 {
13121                let actual = [
13122                    (!activity_events.is_empty()).then_some("activity"),
13123                    (!timer_events.is_empty()).then_some("timer"),
13124                    (!child_events.is_empty()).then_some("child workflow"),
13125                    (!signal_wait_events.is_empty()).then_some("signal wait"),
13126                    (!condition_wait_events.is_empty()).then_some("condition wait"),
13127                    (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13128                    (!side_effect_events.is_empty()).then_some("side effect"),
13129                    (!version_marker_events.is_empty()).then_some("version marker"),
13130                    (!memo_events.is_empty()).then_some("memo upsert"),
13131                ]
13132                .into_iter()
13133                .flatten()
13134                .collect::<Vec<_>>()
13135                .join(" and ");
13136                return Err(invalid_recorded_history(
13137                    "durable_command_sequence_collision",
13138                    sequence,
13139                    "one durable command kind",
13140                    &actual,
13141                    "one workflow sequence records more than one durable command kind",
13142                ));
13143            }
13144
13145            if !activity_events.is_empty() {
13146                let parallel_group_path =
13147                    recorded_parallel_group_path(&activity_events, sequence)?;
13148                let scheduled_count = activity_events
13149                    .iter()
13150                    .filter(|event| event.event_type == "ActivityScheduled")
13151                    .count();
13152                if scheduled_count > 1 {
13153                    return Err(invalid_recorded_history(
13154                        "duplicate_activity_schedule",
13155                        sequence,
13156                        "at most one ActivityScheduled event",
13157                        "multiple ActivityScheduled events",
13158                        "activity history schedules more than one command at one workflow sequence",
13159                    ));
13160                }
13161                let activity_type = activity_events.iter().find_map(|event| {
13162                    event
13163                        .payload
13164                        .get("activity_type")
13165                        .or_else(|| event.payload.get("activity_name"))
13166                        .and_then(Value::as_str)
13167                        .map(str::to_string)
13168                });
13169                if activity_events.iter().filter_map(|event| {
13170                    event
13171                        .payload
13172                        .get("activity_type")
13173                        .or_else(|| event.payload.get("activity_name"))
13174                        .and_then(Value::as_str)
13175                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13176                    return Err(invalid_recorded_history(
13177                        "activity_identity_mismatch",
13178                        sequence,
13179                        activity_type.as_deref().unwrap_or("one activity identity"),
13180                        "conflicting activity identities",
13181                        "activity lifecycle events at one workflow sequence disagree on identity",
13182                    ));
13183                }
13184                let terminal: Vec<_> = activity_events
13185                    .iter()
13186                    .copied()
13187                    .filter(|event| {
13188                        matches!(
13189                            event.event_type.as_str(),
13190                            "ActivityCompleted"
13191                                | "ActivityFailed"
13192                                | "ActivityCancelled"
13193                                | "ActivityTimedOut"
13194                        )
13195                    })
13196                    .collect();
13197                let duplicate_delivery = terminal.first().is_some_and(|first| {
13198                    terminal.iter().all(|event| {
13199                        event.event_type == first.event_type && event.payload == first.payload
13200                    })
13201                });
13202                if terminal.len() > 1 && !duplicate_delivery {
13203                    return Err(invalid_recorded_history(
13204                        "duplicate_activity_terminal_event",
13205                        sequence,
13206                        "at most one terminal activity event",
13207                        "multiple terminal activity events",
13208                        "activity history settles one command more than once",
13209                    ));
13210                }
13211                let outcome = terminal
13212                    .first()
13213                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13214                    .transpose()?;
13215                let options = activity_events
13216                    .iter()
13217                    .find(|event| event.event_type == "ActivityScheduled")
13218                    .and_then(|event| event.payload.get("activity"))
13219                    .and_then(Value::as_object)
13220                    .map(|activity| RecordedActivityOptions {
13221                        task_queue: recorded_optional_string(activity, "queue"),
13222                        execution_mode: recorded_optional_string(activity, "execution_mode"),
13223                        retry_policy: recorded_activity_retry_snapshot(
13224                            activity.get("retry_policy"),
13225                        ),
13226                    });
13227                return Ok(RecordedCommand::Activity {
13228                    sequence,
13229                    activity_type,
13230                    options,
13231                    outcome,
13232                    parallel_group_path,
13233                });
13234            }
13235
13236            if !child_events.is_empty() {
13237                let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13238                let scheduled: Vec<_> = child_events
13239                    .iter()
13240                    .copied()
13241                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
13242                    .collect();
13243                if scheduled.len() != 1 {
13244                    return Err(invalid_recorded_history(
13245                        "child_workflow_schedule_missing_or_duplicate",
13246                        sequence,
13247                        "one ChildWorkflowScheduled event",
13248                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13249                        "child workflow replay requires exactly one recorded schedule event",
13250                    ));
13251                }
13252                let workflow_type = child_events.iter().find_map(|event| {
13253                    event
13254                        .payload
13255                        .get("child_workflow_type")
13256                        .or_else(|| event.payload.get("workflow_type"))
13257                        .and_then(Value::as_str)
13258                        .filter(|value| !value.is_empty())
13259                        .map(str::to_string)
13260                });
13261                if child_events
13262                    .iter()
13263                    .filter_map(|event| {
13264                        event
13265                            .payload
13266                            .get("child_workflow_type")
13267                            .or_else(|| event.payload.get("workflow_type"))
13268                            .and_then(Value::as_str)
13269                    })
13270                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
13271                {
13272                    return Err(invalid_recorded_history(
13273                        "child_workflow_identity_mismatch",
13274                        sequence,
13275                        workflow_type
13276                            .as_deref()
13277                            .unwrap_or("one child workflow type"),
13278                        "conflicting child workflow types",
13279                        "child workflow lifecycle events at one sequence disagree on type",
13280                    ));
13281                }
13282                let mut outcomes = child_workflow_outcomes(
13283                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13284                    fallback_codec,
13285                    parent.clone(),
13286                )?;
13287                let terminal_events = child_events
13288                    .iter()
13289                    .copied()
13290                    .filter(|event| event.event_type.starts_with("ChildRun"))
13291                    .collect::<Vec<_>>();
13292                let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13293                    terminal_events.iter().all(|event| {
13294                        event.event_type == first.event_type && event.payload == first.payload
13295                    })
13296                });
13297                if outcomes.len() > 1 && !duplicate_delivery {
13298                    return Err(invalid_recorded_history(
13299                        "duplicate_child_workflow_terminal_event",
13300                        sequence,
13301                        "at most one terminal child event",
13302                        "multiple terminal child events",
13303                        "child workflow history settles one command more than once",
13304                    ));
13305                }
13306                return Ok(RecordedCommand::ChildWorkflow {
13307                    sequence,
13308                    workflow_type,
13309                    outcome: outcomes.pop(),
13310                    parallel_group_path,
13311                });
13312            }
13313
13314            if !signal_wait_events.is_empty() {
13315                let opened: Vec<_> = signal_wait_events
13316                    .iter()
13317                    .copied()
13318                    .filter(|event| event.event_type == "SignalWaitOpened")
13319                    .collect();
13320                if opened.len() != 1 {
13321                    return Err(invalid_recorded_history(
13322                        "signal_wait_open_missing_or_duplicate",
13323                        sequence,
13324                        "one SignalWaitOpened event",
13325                        &format!("{} SignalWaitOpened events", opened.len()),
13326                        "signal replay requires exactly one canonical wait-open event",
13327                    ));
13328                }
13329
13330                let applied: Vec<_> = signal_wait_events
13331                    .iter()
13332                    .copied()
13333                    .filter(|event| event.event_type == "SignalApplied")
13334                    .collect();
13335                if applied.len() > 1 {
13336                    return Err(invalid_recorded_history(
13337                        "duplicate_signal_wait_apply",
13338                        sequence,
13339                        "at most one SignalApplied event",
13340                        "multiple SignalApplied events",
13341                        "signal history applies one durable wait more than once",
13342                    ));
13343                }
13344
13345                let signal_names = signal_wait_events
13346                    .iter()
13347                    .map(|event| required_signal_wait_name(event, sequence))
13348                    .collect::<Result<Vec<_>>>()?;
13349                let signal_name = signal_names
13350                    .first()
13351                    .expect("signal wait events are not empty")
13352                    .clone();
13353                if signal_names.iter().any(|candidate| candidate != &signal_name) {
13354                    return Err(invalid_recorded_history(
13355                        "signal_wait_identity_mismatch",
13356                        sequence,
13357                        &signal_name,
13358                        "conflicting signal names",
13359                        "signal wait lifecycle events at one workflow sequence disagree on identity",
13360                    ));
13361                }
13362                let value = applied
13363                    .first()
13364                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
13365                    .transpose()?;
13366                return Ok(RecordedCommand::SignalWait {
13367                    sequence,
13368                    signal_name,
13369                    value,
13370                    parallel_group_path: recorded_parallel_group_path(
13371                        &signal_wait_events,
13372                        sequence,
13373                    )?,
13374                });
13375            }
13376
13377            if !condition_wait_events.is_empty() {
13378                return recorded_condition_wait(
13379                    sequence,
13380                    &condition_wait_events,
13381                    events,
13382                );
13383            }
13384
13385            if !search_attribute_events.is_empty() {
13386                if search_attribute_events.len() != 1 {
13387                    return Err(invalid_recorded_history(
13388                        "duplicate_search_attribute_update",
13389                        sequence,
13390                        "one SearchAttributesUpserted event",
13391                        &format!(
13392                            "{} SearchAttributesUpserted events",
13393                            search_attribute_events.len()
13394                        ),
13395                        "search-attribute history records one workflow command more than once",
13396                    ));
13397                }
13398                let payload = &search_attribute_events[0].payload;
13399                let attributes = payload
13400                    .get("attributes")
13401                    .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13402                    .cloned()
13403                    .ok_or_else(|| {
13404                        invalid_recorded_history(
13405                            "search_attribute_update_missing",
13406                            sequence,
13407                            "non-empty attributes object",
13408                            "missing or invalid attributes",
13409                            "search-attribute history is missing its recorded mutation",
13410                        )
13411                    })?;
13412                let attribute_types =
13413                    recorded_search_attribute_types(payload, &attributes, sequence)?;
13414                return Ok(RecordedCommand::SearchAttributes {
13415                    sequence,
13416                    attributes,
13417                    attribute_types,
13418                });
13419            }
13420
13421            if !side_effect_events.is_empty() {
13422                if side_effect_events.len() != 1 {
13423                    return Err(invalid_recorded_history(
13424                        "duplicate_side_effect_record",
13425                        sequence,
13426                        "one SideEffectRecorded event",
13427                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
13428                        "side-effect history records one workflow command more than once",
13429                    ));
13430                }
13431                let event = side_effect_events[0];
13432                let result = event.payload.get("result").ok_or_else(|| {
13433                    invalid_recorded_history(
13434                        "side_effect_result_missing",
13435                        sequence,
13436                        "recorded result payload",
13437                        "missing result",
13438                        "side-effect history is missing its recorded value",
13439                    )
13440                })?;
13441                let has_published_envelope = result.as_str().is_some()
13442                    || result.as_object().is_some_and(|envelope| {
13443                        envelope.get("codec").and_then(Value::as_str).is_some()
13444                            && envelope.get("blob").and_then(Value::as_str).is_some()
13445                    });
13446                if !has_published_envelope {
13447                    return Err(invalid_recorded_history(
13448                        "side_effect_payload_malformed",
13449                        sequence,
13450                        "payload blob or {codec, blob} envelope",
13451                        &result.to_string(),
13452                        "side-effect history result does not use a published payload envelope",
13453                    ));
13454                }
13455                let codec = event
13456                    .payload
13457                    .get("payload_codec")
13458                    .and_then(Value::as_str)
13459                    .unwrap_or(fallback_codec);
13460                let value = decode_wire_avro_value(result, codec).map_err(|error| {
13461                    if error.to_string().contains("unsupported_payload_codec") {
13462                        return error;
13463                    }
13464
13465                    invalid_recorded_history(
13466                        "side_effect_payload_incompatible",
13467                        sequence,
13468                        &format!("valid {codec} payload envelope"),
13469                        &error.to_string(),
13470                        "side-effect history payload cannot be decoded with its recorded codec",
13471                    )
13472                })?;
13473                return Ok(RecordedCommand::SideEffect { sequence, value });
13474            }
13475
13476            if !version_marker_events.is_empty() {
13477                if version_marker_events.len() != 1 {
13478                    return Err(invalid_recorded_history(
13479                        "duplicate_version_marker_record",
13480                        sequence,
13481                        "one VersionMarkerRecorded event",
13482                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13483                        "version-marker history records one workflow command more than once",
13484                    ));
13485                }
13486                let payload = &version_marker_events[0].payload;
13487                let change_id = payload
13488                    .get("change_id")
13489                    .and_then(Value::as_str)
13490                    .filter(|value| !value.is_empty())
13491                    .map(str::to_string)
13492                    .ok_or_else(|| {
13493                        invalid_recorded_history(
13494                            "version_marker_field_missing",
13495                            sequence,
13496                            "non-empty change_id",
13497                            "missing or invalid change_id",
13498                            "version-marker history is missing its stable change ID",
13499                        )
13500                    })?;
13501                let version = required_version_i32(payload, "version", sequence)?;
13502                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13503                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13504                if min_supported > max_supported || version < min_supported || version > max_supported {
13505                    return Err(invalid_recorded_history(
13506                        "version_marker_history_range_invalid",
13507                        sequence,
13508                        "min_supported <= version <= max_supported",
13509                        &format!("{min_supported} <= {version} <= {max_supported}"),
13510                        "recorded version marker contains an internally incompatible range",
13511                    ));
13512                }
13513                return Ok(RecordedCommand::VersionMarker {
13514                    sequence,
13515                    change_id,
13516                    version,
13517                });
13518            }
13519
13520            if !memo_events.is_empty() {
13521                if memo_events.len() != 1 {
13522                    return Err(invalid_recorded_history(
13523                        "duplicate_memo_upsert_record",
13524                        sequence,
13525                        "one MemoUpserted event",
13526                        &format!("{} MemoUpserted events", memo_events.len()),
13527                        "memo history records one workflow update more than once",
13528                    ));
13529                }
13530                let payload = &memo_events[0].payload;
13531                let entries = payload.get("entries").cloned().ok_or_else(|| {
13532                    invalid_recorded_history(
13533                        "memo_entries_missing",
13534                        sequence,
13535                        "memo entries object",
13536                        "missing entries",
13537                        "MemoUpserted history is missing replay identity entries",
13538                    )
13539                })?;
13540                let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13541                    invalid_recorded_history(
13542                        "memo_entries_invalid",
13543                        sequence,
13544                        "valid canonical memo entries",
13545                        &error.to_string(),
13546                        "MemoUpserted history contains invalid replay identity entries",
13547                    )
13548                })?;
13549                let merged = payload.get("merged").cloned().ok_or_else(|| {
13550                    invalid_recorded_history(
13551                        "memo_merged_projection_missing",
13552                        sequence,
13553                        "merged memo projection",
13554                        "missing merged",
13555                        "MemoUpserted history is missing its merged projection",
13556                    )
13557                })?;
13558                decode_memo_history_map(&merged, false).map_err(|error| {
13559                    invalid_recorded_history(
13560                        "memo_merged_projection_invalid",
13561                        sequence,
13562                        "valid merged memo projection",
13563                        &error.to_string(),
13564                        "MemoUpserted history contains an invalid merged projection",
13565                    )
13566                })?;
13567
13568                return Ok(RecordedCommand::Memo { sequence, entries });
13569            }
13570            let scheduled: Vec<_> = timer_events
13571                .iter()
13572                .copied()
13573                .filter(|event| event.event_type == "TimerScheduled")
13574                .collect();
13575            let fired: Vec<_> = timer_events
13576                .iter()
13577                .copied()
13578                .filter(|event| event.event_type == "TimerFired")
13579                .collect();
13580            if scheduled.len() != 1 {
13581                return Err(invalid_recorded_history(
13582                    "timer_schedule_missing_or_duplicate",
13583                    sequence,
13584                    "one TimerScheduled event",
13585                    &format!("{} TimerScheduled events", scheduled.len()),
13586                    "timer replay requires exactly one recorded schedule event",
13587                ));
13588            }
13589            if fired.len() > 1 {
13590                return Err(invalid_recorded_history(
13591                    "duplicate_timer_fire",
13592                    sequence,
13593                    "at most one TimerFired event",
13594                    "multiple TimerFired events",
13595                    "timer history contains more than one fire event for a workflow sequence",
13596                ));
13597            }
13598
13599            let scheduled = scheduled[0];
13600            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13601            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13602            if let Some(fired) = fired.first() {
13603                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13604                if fired_timer_id != timer_id {
13605                    return Err(invalid_recorded_history(
13606                        "timer_identity_mismatch",
13607                        sequence,
13608                        &timer_id,
13609                        &fired_timer_id,
13610                        "TimerFired does not correspond to the recorded TimerScheduled event",
13611                    ));
13612                }
13613                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13614                if fired_delay != delay_seconds {
13615                    return Err(invalid_recorded_history(
13616                        "timer_history_delay_mismatch",
13617                        sequence,
13618                        &delay_seconds.to_string(),
13619                        &fired_delay.to_string(),
13620                        "TimerScheduled and TimerFired record different delays",
13621                    ));
13622                }
13623            }
13624
13625            Ok(RecordedCommand::Timer {
13626                sequence,
13627                delay_seconds,
13628                fired: !fired.is_empty(),
13629                parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13630            })
13631        })
13632        .collect::<Result<_>>()?;
13633
13634    let mut marker_sequences = HashMap::new();
13635    for command in &commands {
13636        if let RecordedCommand::VersionMarker {
13637            sequence,
13638            change_id,
13639            ..
13640        } = command
13641        {
13642            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13643                return Err(invalid_recorded_history(
13644                    "duplicate_version_marker",
13645                    *sequence,
13646                    &format!("one marker for change ID {change_id:?}"),
13647                    &format!("markers at sequences {first_sequence} and {sequence}"),
13648                    "workflow history contains duplicate markers for one stable change ID",
13649                ));
13650            }
13651        }
13652    }
13653
13654    Ok(commands)
13655}
13656
13657fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13658    payload
13659        .get(field)
13660        .and_then(Value::as_i64)
13661        .and_then(|value| i32::try_from(value).ok())
13662        .ok_or_else(|| {
13663            invalid_recorded_history(
13664                "version_marker_field_missing",
13665                sequence,
13666                &format!("integer {field}"),
13667                "missing or out-of-range integer",
13668                "version-marker history is missing a required integer field",
13669            )
13670        })
13671}
13672
13673fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13674    event
13675        .payload
13676        .get("sequence")
13677        .or_else(|| event.payload.get("workflow_sequence"))
13678        .or_else(|| event.raw.get("sequence"))
13679        .or_else(|| event.raw.get("workflow_sequence"))
13680        .and_then(value_as_u64)
13681}
13682
13683fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13684    matches!(
13685        event
13686            .payload
13687            .get("timer_kind")
13688            .or_else(|| event.raw.get("timer_kind"))
13689            .and_then(Value::as_str),
13690        Some("condition_timeout" | "signal_timeout")
13691    )
13692}
13693
13694fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13695    matches!(
13696        event.event_type.as_str(),
13697        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13698    )
13699}
13700
13701fn recorded_condition_wait(
13702    sequence: u64,
13703    condition_events: &[&HistoryEvent],
13704    all_events: &[HistoryEvent],
13705) -> Result<RecordedCommand> {
13706    let opened = condition_events
13707        .iter()
13708        .copied()
13709        .filter(|event| event.event_type == "ConditionWaitOpened")
13710        .collect::<Vec<_>>();
13711    if opened.len() != 1 {
13712        return Err(invalid_recorded_history(
13713            "condition_wait_open_missing_or_duplicate",
13714            sequence,
13715            "one ConditionWaitOpened event",
13716            &format!("{} ConditionWaitOpened events", opened.len()),
13717            "condition replay requires exactly one canonical wait-open event",
13718        ));
13719    }
13720    let terminal = condition_events
13721        .iter()
13722        .copied()
13723        .filter(|event| {
13724            matches!(
13725                event.event_type.as_str(),
13726                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13727            )
13728        })
13729        .collect::<Vec<_>>();
13730    if terminal.len() > 1 {
13731        return Err(invalid_recorded_history(
13732            "duplicate_condition_wait_terminal_event",
13733            sequence,
13734            "at most one condition terminal event",
13735            "multiple condition terminal events",
13736            "condition history settles one durable wait more than once",
13737        ));
13738    }
13739
13740    let opened = opened[0];
13741    let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13742    let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13743    for event in condition_events
13744        .iter()
13745        .copied()
13746        .filter(|event| !std::ptr::eq(*event, opened))
13747    {
13748        let event_wait_id = required_condition_wait_id(event, sequence)?;
13749        if event_wait_id != condition_wait_id {
13750            return Err(invalid_recorded_history(
13751                "condition_wait_id_mismatch",
13752                sequence,
13753                &condition_wait_id,
13754                &event_wait_id,
13755                "condition lifecycle events at one sequence disagree on wait identity",
13756            ));
13757        }
13758        let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13759        if event_occurrence_id != occurrence_id {
13760            return Err(invalid_recorded_history(
13761                "condition_wait_occurrence_history_mismatch",
13762                sequence,
13763                &occurrence_id,
13764                &event_occurrence_id,
13765                "condition lifecycle events at one sequence disagree on authored occurrence identity",
13766            ));
13767        }
13768    }
13769
13770    let condition_key = optional_non_empty_history_string(opened, "condition_key");
13771    let predicate_identity = opened
13772        .payload
13773        .get("condition_definition_fingerprint")
13774        .and_then(Value::as_str)
13775        .filter(|value| !value.is_empty())
13776        .map(str::to_string)
13777        .ok_or_else(|| {
13778            invalid_recorded_history(
13779                "condition_wait_predicate_fingerprint_missing",
13780                sequence,
13781                "non-empty condition_definition_fingerprint",
13782                &opened.event_type,
13783                "canonical condition history is missing its predicate identity",
13784            )
13785        })?;
13786    let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13787    for event in condition_events
13788        .iter()
13789        .copied()
13790        .filter(|event| !std::ptr::eq(*event, opened))
13791    {
13792        for (field, opened_value) in [
13793            ("condition_key", condition_key.as_deref()),
13794            (
13795                "condition_definition_fingerprint",
13796                Some(predicate_identity.as_str()),
13797            ),
13798        ] {
13799            if let Some(value) = optional_non_empty_history_string(event, field) {
13800                if opened_value.is_some_and(|opened_value| opened_value != value) {
13801                    return Err(invalid_recorded_history(
13802                        "condition_wait_definition_history_mismatch",
13803                        sequence,
13804                        opened_value.unwrap_or_default(),
13805                        &value,
13806                        "condition lifecycle events disagree on the recorded definition",
13807                    ));
13808                }
13809            }
13810        }
13811        if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13812            if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13813                return Err(invalid_recorded_history(
13814                    "condition_wait_definition_history_mismatch",
13815                    sequence,
13816                    &format!("{}s", timeout_seconds.unwrap_or_default()),
13817                    &format!("{event_timeout}s"),
13818                    "condition lifecycle events disagree on the recorded timeout",
13819                ));
13820            }
13821        }
13822    }
13823
13824    let timeout_timer_events = all_events
13825        .iter()
13826        .filter(|event| {
13827            matches!(
13828                event.event_type.as_str(),
13829                "TimerScheduled" | "TimerCancelled" | "TimerFired"
13830            ) && event.payload.get("timer_kind").and_then(Value::as_str)
13831                == Some("condition_timeout")
13832                && event
13833                    .payload
13834                    .get("condition_wait_id")
13835                    .and_then(Value::as_str)
13836                    == Some(condition_wait_id.as_str())
13837        })
13838        .collect::<Vec<_>>();
13839    let scheduled = timeout_timer_events
13840        .iter()
13841        .copied()
13842        .filter(|event| event.event_type == "TimerScheduled")
13843        .collect::<Vec<_>>();
13844    let fired = timeout_timer_events
13845        .iter()
13846        .copied()
13847        .filter(|event| event.event_type == "TimerFired")
13848        .collect::<Vec<_>>();
13849    if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13850        return Err(invalid_recorded_history(
13851            "condition_wait_timeout_history_invalid",
13852            sequence,
13853            "one timeout schedule and at most one fire",
13854            &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13855            "condition timeout history has a missing or duplicate lifecycle event",
13856        ));
13857    }
13858    if let Some(scheduled) = scheduled.first() {
13859        let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13860        let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13861        if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13862            return Err(invalid_recorded_history(
13863                "condition_wait_timeout_delay_mismatch",
13864                sequence,
13865                &format!("{}s", timeout_seconds.unwrap_or_default()),
13866                &format!("{delay_seconds}s"),
13867                "condition timeout timer differs from the wait definition",
13868            ));
13869        }
13870        if let Some(fired) = fired.first() {
13871            let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13872            let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13873            if fired_timer_id != timer_id || fired_delay != delay_seconds {
13874                return Err(invalid_recorded_history(
13875                    "condition_wait_timeout_identity_mismatch",
13876                    sequence,
13877                    &format!("{timer_id}:{delay_seconds}s"),
13878                    &format!("{fired_timer_id}:{fired_delay}s"),
13879                    "condition timeout fire does not match its durable schedule",
13880                ));
13881            }
13882        }
13883    }
13884
13885    let result = terminal.first().map(|event| {
13886        if event.event_type == "ConditionWaitTimedOut" {
13887            ConditionWaitResult::TimedOut
13888        } else {
13889            ConditionWaitResult::Satisfied
13890        }
13891    });
13892    let result = if !fired.is_empty() {
13893        if result == Some(ConditionWaitResult::Satisfied) {
13894            return Err(invalid_recorded_history(
13895                "condition_wait_terminal_conflict",
13896                sequence,
13897                "one satisfied or timed-out outcome",
13898                "satisfied event and fired timeout",
13899                "condition history records conflicting terminal outcomes",
13900            ));
13901        }
13902        Some(ConditionWaitResult::TimedOut)
13903    } else {
13904        result
13905    };
13906
13907    Ok(RecordedCommand::ConditionWait {
13908        sequence,
13909        occurrence_id,
13910        condition_key,
13911        predicate_identity,
13912        timeout_seconds,
13913        result,
13914        parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13915    })
13916}
13917
13918fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13919    event
13920        .payload
13921        .get("condition_wait_occurrence_id")
13922        .and_then(Value::as_str)
13923        .filter(|value| !value.is_empty())
13924        .map(str::to_string)
13925        .ok_or_else(|| {
13926            invalid_recorded_history(
13927                "condition_wait_occurrence_id_missing",
13928                sequence,
13929                "non-empty condition_wait_occurrence_id",
13930                &event.event_type,
13931                "condition history is missing authored occurrence identity",
13932            )
13933        })
13934}
13935
13936fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13937    event
13938        .payload
13939        .get("condition_wait_id")
13940        .and_then(Value::as_str)
13941        .filter(|value| !value.is_empty())
13942        .map(str::to_string)
13943        .ok_or_else(|| {
13944            invalid_recorded_history(
13945                "condition_wait_id_missing",
13946                sequence,
13947                "non-empty condition_wait_id",
13948                &event.event_type,
13949                "canonical condition history is missing its durable wait identity",
13950            )
13951        })
13952}
13953
13954fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13955    event
13956        .payload
13957        .get(field)
13958        .and_then(Value::as_str)
13959        .filter(|value| !value.is_empty())
13960        .map(str::to_string)
13961}
13962
13963fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13964    match event.payload.get(field) {
13965        None | Some(Value::Null) => Ok(None),
13966        Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13967            invalid_recorded_history(
13968                "condition_wait_definition_invalid",
13969                sequence,
13970                &format!("non-negative integer {field}"),
13971                &value.to_string(),
13972                "condition history contains an invalid numeric definition field",
13973            )
13974        }),
13975    }
13976}
13977
13978fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13979    event
13980        .payload
13981        .get("signal_name")
13982        .or_else(|| event.raw.get("signal_name"))
13983        .and_then(Value::as_str)
13984        .filter(|value| !value.is_empty())
13985        .map(str::to_string)
13986        .ok_or_else(|| {
13987            invalid_recorded_history(
13988                "signal_wait_name_missing",
13989                sequence,
13990                "non-empty signal_name",
13991                &event.event_type,
13992                "canonical signal-wait history is missing its signal identity",
13993            )
13994        })
13995}
13996
13997fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13998    matches!(
13999        event.event_type.as_str(),
14000        "SignalWaitOpened" | "SignalApplied"
14001    )
14002}
14003
14004fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
14005    event
14006        .payload
14007        .get(field)
14008        .and_then(Value::as_str)
14009        .filter(|value| !value.is_empty())
14010        .map(str::to_string)
14011        .ok_or_else(|| {
14012            invalid_recorded_history(
14013                "timer_history_field_missing",
14014                sequence,
14015                field,
14016                &event.event_type,
14017                "timer history is missing a required identity field",
14018            )
14019        })
14020}
14021
14022fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
14023    event
14024        .payload
14025        .get(field)
14026        .and_then(value_as_u64)
14027        .ok_or_else(|| {
14028            invalid_recorded_history(
14029                "timer_history_field_missing",
14030                sequence,
14031                field,
14032                &event.event_type,
14033                "timer history is missing a required numeric field",
14034            )
14035        })
14036}
14037
14038fn recorded_search_attribute_types(
14039    payload: &Value,
14040    attributes: &Value,
14041    sequence: u64,
14042) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
14043    let Some(raw_types) = payload.get("attribute_types") else {
14044        // This is the explicit compatibility rule for histories recorded
14045        // before typed identity was persisted. Values still constrain replay;
14046        // the unknown type snapshot does not assert a typed match.
14047        return Ok(RecordedSnapshotValue::Unknown);
14048    };
14049    let Some(raw_types) = raw_types.as_object() else {
14050        return Err(invalid_recorded_history(
14051            "search_attribute_types_malformed",
14052            sequence,
14053            "canonical attribute type map",
14054            &raw_types.to_string(),
14055            "search-attribute history contains malformed type identity",
14056        ));
14057    };
14058    let attribute_keys = attributes
14059        .as_object()
14060        .expect("recorded search attributes were validated as an object");
14061    let mut types = BTreeMap::new();
14062    for (key, value) in raw_types {
14063        let Some(attribute_type) = value.as_str() else {
14064            return Err(invalid_recorded_history(
14065                "search_attribute_types_malformed",
14066                sequence,
14067                "canonical string type name",
14068                &value.to_string(),
14069                "search-attribute history contains a non-string type identity",
14070            ));
14071        };
14072        if !attribute_keys.contains_key(key)
14073            || !matches!(
14074                attribute_type,
14075                "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
14076            )
14077        {
14078            return Err(invalid_recorded_history(
14079                "search_attribute_types_malformed",
14080                sequence,
14081                "canonical types for keys present in attributes",
14082                &format!("{key}:{attribute_type}"),
14083                "search-attribute history contains unsupported or orphaned type identity",
14084            ));
14085        }
14086        types.insert(key.clone(), attribute_type.to_string());
14087    }
14088    Ok(RecordedSnapshotValue::Known(types))
14089}
14090
14091fn invalid_recorded_history(
14092    reason: &str,
14093    sequence: u64,
14094    expected: &str,
14095    actual: &str,
14096    message: &str,
14097) -> Error {
14098    Error::NonDeterministicReplay(ReplayFailure::new(
14099        reason,
14100        Some(sequence),
14101        Some(expected.to_string()),
14102        Some(actual.to_string()),
14103        message,
14104    ))
14105}
14106
14107type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
14108
14109fn activity_outcome(
14110    event: &HistoryEvent,
14111    fallback_codec: &str,
14112    recorded_activity_type: Option<String>,
14113) -> Result<ActivityOutcome> {
14114    if event.event_type == "ActivityCompleted" {
14115        let codec = event
14116            .payload
14117            .get("payload_codec")
14118            .and_then(Value::as_str)
14119            .unwrap_or(fallback_codec);
14120        return Ok(Ok(decode_wire_avro_value(
14121            event.payload.get("result").unwrap_or(&Value::Null),
14122            codec,
14123        )?));
14124    }
14125
14126    let payload = &event.payload;
14127    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14128        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14129        "ActivityCancelled" => (
14130            ActivityFailureKind::Cancelled,
14131            "cancelled",
14132            "activity was cancelled",
14133        ),
14134        "ActivityTimedOut" => (
14135            ActivityFailureKind::TimedOut,
14136            "timeout",
14137            "activity timed out",
14138        ),
14139        _ => unreachable!("activity_outcome is called only for terminal activity events"),
14140    };
14141    let exception = payload
14142        .get("exception")
14143        .filter(|value| !value.is_null())
14144        .cloned();
14145    let failure_category = payload_string(payload, "failure_category");
14146    let timeout_kind = payload_string(payload, "timeout_kind");
14147    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14148        ActivityFailureKind::Failed => failure_category
14149            .clone()
14150            .unwrap_or_else(|| fallback_reason.to_string()),
14151        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14152        ActivityFailureKind::TimedOut => timeout_kind
14153            .clone()
14154            .unwrap_or_else(|| fallback_reason.to_string()),
14155    });
14156    let message = payload_string(payload, "message")
14157        .or_else(|| {
14158            exception
14159                .as_ref()
14160                .and_then(|value| payload_string(value, "message"))
14161        })
14162        .unwrap_or_else(|| fallback_message.to_string());
14163
14164    Ok(Err(ActivityFailure {
14165        kind,
14166        reason,
14167        message,
14168        activity_execution_id: payload_string(payload, "activity_execution_id"),
14169        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14170        activity_type: payload_string(payload, "activity_type")
14171            .or_else(|| payload_string(payload, "activity_name"))
14172            .or(recorded_activity_type),
14173        activity_class: payload_string(payload, "activity_class"),
14174        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14175        failure_id: payload_string(payload, "failure_id"),
14176        failure_category,
14177        timeout_kind,
14178        non_retryable: payload
14179            .get("non_retryable")
14180            .and_then(Value::as_bool)
14181            .unwrap_or(false),
14182        exception_type: payload_string(payload, "exception_type").or_else(|| {
14183            exception
14184                .as_ref()
14185                .and_then(|value| payload_string(value, "type"))
14186        }),
14187        exception_class: payload_string(payload, "exception_class").or_else(|| {
14188            exception
14189                .as_ref()
14190                .and_then(|value| payload_string(value, "class"))
14191        }),
14192        code: payload
14193            .get("code")
14194            .filter(|value| !value.is_null())
14195            .cloned(),
14196        exception,
14197    }))
14198}
14199
14200type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14201
14202fn child_workflow_outcomes(
14203    events: &[HistoryEvent],
14204    fallback_codec: &str,
14205    parent: WorkflowIdentity,
14206) -> Result<Vec<ChildWorkflowOutcome>> {
14207    let mut outcomes = Vec::new();
14208
14209    for event in events {
14210        let kind = match event.event_type.as_str() {
14211            "ChildRunCompleted" => None,
14212            "ChildRunFailed" => Some((
14213                ChildWorkflowFailureKind::Failed,
14214                "child_workflow",
14215                "child workflow failed",
14216            )),
14217            "ChildRunCancelled" => Some((
14218                ChildWorkflowFailureKind::Cancelled,
14219                "cancelled",
14220                "child workflow was cancelled",
14221            )),
14222            "ChildRunTerminated" => Some((
14223                ChildWorkflowFailureKind::Terminated,
14224                "terminated",
14225                "child workflow was terminated",
14226            )),
14227            _ => continue,
14228        };
14229        let payload = &event.payload;
14230        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14231        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14232        let child_workflow_type = payload_string(payload, "child_workflow_type");
14233
14234        if let Some((kind, reason, fallback_message)) = kind {
14235            let exception = payload
14236                .get("exception")
14237                .filter(|value| !value.is_null())
14238                .cloned();
14239            let message = payload_string(payload, "message")
14240                .or_else(|| {
14241                    exception
14242                        .as_ref()
14243                        .and_then(|value| payload_string(value, "message"))
14244                })
14245                .unwrap_or_else(|| fallback_message.to_string());
14246            let exception_type = payload_string(payload, "exception_type").or_else(|| {
14247                exception
14248                    .as_ref()
14249                    .and_then(|value| payload_string(value, "type"))
14250            });
14251            let exception_class = payload_string(payload, "exception_class").or_else(|| {
14252                exception
14253                    .as_ref()
14254                    .and_then(|value| payload_string(value, "class"))
14255            });
14256            outcomes.push(Err(ChildWorkflowFailure {
14257                kind,
14258                reason: reason.to_string(),
14259                message,
14260                parent_workflow_id: parent.workflow_id.clone(),
14261                parent_workflow_run_id: parent.run_id.clone(),
14262                child_workflow_id,
14263                child_workflow_run_id,
14264                child_workflow_type,
14265                failure_id: payload_string(payload, "failure_id"),
14266                failure_category: payload_string(payload, "failure_category"),
14267                exception_type,
14268                exception_class,
14269                non_retryable: payload
14270                    .get("non_retryable")
14271                    .and_then(Value::as_bool)
14272                    .unwrap_or(false),
14273                code: payload
14274                    .get("code")
14275                    .filter(|value| !value.is_null())
14276                    .cloned(),
14277                exception,
14278            }));
14279            continue;
14280        }
14281
14282        let codec = payload
14283            .get("payload_codec")
14284            .and_then(Value::as_str)
14285            .unwrap_or(fallback_codec);
14286        let result = payload
14287            .get("result")
14288            .or_else(|| payload.get("output"))
14289            .unwrap_or(&Value::Null);
14290        outcomes.push(Ok(ChildWorkflowAvroResult {
14291            parent: parent.clone(),
14292            child: WorkflowIdentity {
14293                workflow_id: child_workflow_id,
14294                run_id: child_workflow_run_id,
14295            },
14296            child_workflow_type,
14297            result: decode_wire_avro_value(result, codec)?,
14298        }));
14299    }
14300
14301    Ok(outcomes)
14302}
14303
14304fn payload_string(payload: &Value, key: &str) -> Option<String> {
14305    payload
14306        .get(key)
14307        .and_then(Value::as_str)
14308        .filter(|value| !value.is_empty())
14309        .map(str::to_string)
14310}
14311
14312fn workflow_failure_command(error: &Error) -> Value {
14313    let (exception_type, exception_class, properties) = match error {
14314        Error::ActivityFailed(failure) => (
14315            match failure.kind {
14316                ActivityFailureKind::Failed => "ActivityFailed",
14317                ActivityFailureKind::Cancelled => "ActivityCancelled",
14318                ActivityFailureKind::TimedOut => "ActivityTimedOut",
14319            },
14320            "durable_workflow::ActivityFailure",
14321            json!({
14322                "reason": failure.reason,
14323                "activity_execution_id": failure.activity_execution_id,
14324                "activity_attempt_id": failure.activity_attempt_id,
14325                "activity_type": failure.activity_type,
14326                "activity_class": failure.activity_class,
14327                "attempt_number": failure.attempt_number,
14328                "failure_id": failure.failure_id,
14329                "failure_category": failure.failure_category,
14330                "timeout_kind": failure.timeout_kind,
14331                "activity_non_retryable": failure.non_retryable,
14332                "activity_exception_type": failure.exception_type,
14333                "activity_exception_class": failure.exception_class,
14334                "activity_code": failure.code,
14335                "activity_exception": failure.exception,
14336            }),
14337        ),
14338        Error::ChildWorkflowFailed(failure) => (
14339            match failure.kind {
14340                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14341                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14342                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14343            },
14344            "durable_workflow::ChildWorkflowFailure",
14345            json!({
14346                "reason": failure.reason,
14347                "parent_workflow_id": failure.parent_workflow_id,
14348                "parent_workflow_run_id": failure.parent_workflow_run_id,
14349                "child_workflow_id": failure.child_workflow_id,
14350                "child_workflow_run_id": failure.child_workflow_run_id,
14351                "child_workflow_type": failure.child_workflow_type,
14352                "failure_id": failure.failure_id,
14353                "failure_category": failure.failure_category,
14354                "child_exception_type": failure.exception_type,
14355                "child_exception_class": failure.exception_class,
14356                "child_non_retryable": failure.non_retryable,
14357                "child_code": failure.code,
14358                "child_exception": failure.exception,
14359            }),
14360        ),
14361        Error::ParallelFailed(failure) => (
14362            "ParallelFailed",
14363            "durable_workflow::ParallelFailure",
14364            json!({
14365                "parallel_group_id": failure.group_id,
14366                "parallel_member_path": failure.member_path,
14367                "parallel_group_path": failure.group_path,
14368                "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14369                "cause_type": workflow_error_type(&failure.cause),
14370                "cause_message": failure.cause.to_string(),
14371            }),
14372        ),
14373        Error::SagaCompensationFailed(failure) => (
14374            "SagaCompensationFailed",
14375            "durable_workflow::SagaCompensationFailure",
14376            json!({
14377                "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14378                "initiating_failure_message": failure.initiating_failure.to_string(),
14379                "compensation_activity_type": failure.compensation_activity_type,
14380                "compensation_registration_order": failure.compensation_registration_order,
14381                "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14382                "compensation_failure_message": failure.compensation_failure.to_string(),
14383            }),
14384        ),
14385        Error::WorkflowCancellationRequested(_) => (
14386            "WorkflowCancellationRequested",
14387            "durable_workflow::WorkflowCancellationRequested",
14388            json!({"reason": "cancelled"}),
14389        ),
14390        Error::NonDeterministicReplay(_) => (
14391            "NonDeterministicReplay",
14392            "durable_workflow::Error",
14393            Value::Null,
14394        ),
14395        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14396    };
14397    let non_retryable = match error {
14398        Error::ActivityFailed(failure) => failure.non_retryable,
14399        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14400        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14401        Error::SagaCompensationFailed(failure) => {
14402            workflow_error_non_retryable(&failure.compensation_failure)
14403        }
14404        Error::WorkflowCancellationRequested(_) => true,
14405        Error::NonDeterministicReplay(_) => true,
14406        _ => false,
14407    };
14408
14409    json!({
14410        "type": "fail_workflow",
14411        "message": error.to_string(),
14412        "exception_type": exception_type,
14413        "exception_class": exception_class,
14414        "non_retryable": non_retryable,
14415        "exception": {
14416            "type": exception_type,
14417            "class": exception_class,
14418            "message": error.to_string(),
14419            "properties": properties,
14420        }
14421    })
14422}
14423
14424fn workflow_error_type(error: &Error) -> &'static str {
14425    match error {
14426        Error::ActivityFailed(failure) => match failure.kind {
14427            ActivityFailureKind::Failed => "ActivityFailed",
14428            ActivityFailureKind::Cancelled => "ActivityCancelled",
14429            ActivityFailureKind::TimedOut => "ActivityTimedOut",
14430        },
14431        Error::ChildWorkflowFailed(failure) => match failure.kind {
14432            ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14433            ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14434            ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14435        },
14436        Error::ParallelFailed(_) => "ParallelFailed",
14437        Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14438        Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14439        Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14440        _ => "RustWorkflowError",
14441    }
14442}
14443
14444fn workflow_error_non_retryable(error: &Error) -> bool {
14445    match error {
14446        Error::ActivityFailed(failure) => failure.non_retryable,
14447        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14448        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14449        Error::SagaCompensationFailed(failure) => {
14450            workflow_error_non_retryable(&failure.compensation_failure)
14451        }
14452        Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14453        _ => false,
14454    }
14455}
14456
14457fn workflow_task_integrity_error(error: &Error) -> bool {
14458    matches!(
14459        error,
14460        Error::NonDeterministicReplay(_)
14461            | Error::Protocol(_)
14462            | Error::MissingWorkflowCommandIdentity
14463            | Error::WorkflowStatePoisoned
14464    )
14465}
14466
14467fn decode_signal_event_arguments(
14468    event: &HistoryEvent,
14469    fallback_codec: &str,
14470) -> Result<Vec<AvroValue>> {
14471    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14472    validate_payload_codec(codec)?;
14473    let raw = signal_history_payload(&event.payload);
14474    let decoded = match raw.filter(|value| !value.is_null()) {
14475        Some(value) => decode_wire_avro_value(value, codec)?,
14476        None => AvroValue::Array(Vec::new()),
14477    };
14478    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14479        unreachable!("normalize_avro_arguments always returns an array");
14480    };
14481    Ok(arguments)
14482}
14483
14484fn decode_update_event_arguments(
14485    event: &HistoryEvent,
14486    fallback_codec: &str,
14487) -> Result<Vec<AvroValue>> {
14488    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14489    validate_payload_codec(codec)?;
14490    let decoded = match event
14491        .payload
14492        .get("arguments")
14493        .filter(|value| !value.is_null())
14494    {
14495        Some(value) => decode_wire_avro_value(value, codec)?,
14496        None => AvroValue::Array(Vec::new()),
14497    };
14498    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14499        unreachable!("normalize_avro_arguments always returns an array");
14500    };
14501    Ok(arguments)
14502}
14503
14504fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14505    let Some(export_events) = task
14506        .history_export
14507        .as_ref()
14508        .and_then(|export| export.get("history_events"))
14509        .and_then(Value::as_array)
14510    else {
14511        return Ok(());
14512    };
14513
14514    if export_events.len() > task.history_events.len() {
14515        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14516    }
14517
14518    Ok(())
14519}
14520
14521fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14522    let Some(export) = task.history_export.as_ref() else {
14523        return Ok(());
14524    };
14525    let signals = export
14526        .get("signals")
14527        .and_then(Value::as_array)
14528        .cloned()
14529        .unwrap_or_default();
14530    let activities = export
14531        .get("activities")
14532        .and_then(Value::as_array)
14533        .cloned()
14534        .unwrap_or_default();
14535    let export_codec = export
14536        .get("payloads")
14537        .and_then(|payloads| payloads.get("codec"))
14538        .and_then(Value::as_str)
14539        .unwrap_or(&task.payload_codec)
14540        .to_string();
14541    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14542
14543    for event in &mut task.history_events {
14544        if event.event_type == "ActivityCompleted" {
14545            let sequence = event
14546                .payload
14547                .get("sequence")
14548                .or_else(|| event.payload.get("workflow_sequence"))
14549                .and_then(value_as_u64);
14550            let Some(activity) = sequence.and_then(|sequence| {
14551                activities.iter().find(|activity| {
14552                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14553                })
14554            }) else {
14555                continue;
14556            };
14557            let Some(payload) = event.payload.as_object_mut() else {
14558                continue;
14559            };
14560            if missing_payload(payload.get("result")) {
14561                if let Some(result) = activity
14562                    .get("result")
14563                    .filter(|value| !missing_payload(Some(value)))
14564                {
14565                    payload.insert("result".to_string(), result.clone());
14566                }
14567            }
14568            for field in ["payload_codec", "activity_type"] {
14569                if payload
14570                    .get(field)
14571                    .and_then(Value::as_str)
14572                    .unwrap_or_default()
14573                    .is_empty()
14574                {
14575                    if let Some(value) = activity.get(field) {
14576                        payload.insert(field.to_string(), value.clone());
14577                    }
14578                }
14579            }
14580            continue;
14581        }
14582
14583        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14584            continue;
14585        }
14586        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14587        let command_id = event
14588            .payload
14589            .get("workflow_command_id")
14590            .or_else(|| event.raw.get("workflow_command_id"))
14591            .and_then(Value::as_str);
14592        let signal_name = event
14593            .payload
14594            .get("signal_name")
14595            .and_then(Value::as_str)
14596            .unwrap_or_default()
14597            .to_string();
14598        let matched = signals
14599            .iter()
14600            .find(|signal| {
14601                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14602            })
14603            .or_else(|| {
14604                signals.iter().find(|signal| {
14605                    command_id.is_some()
14606                        && signal.get("command_id").and_then(Value::as_str) == command_id
14607                })
14608            })
14609            .or_else(|| {
14610                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14611                let signal = signals
14612                    .iter()
14613                    .filter(|signal| {
14614                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14615                    })
14616                    .nth(*offset);
14617                if signal.is_some() {
14618                    *offset += 1;
14619                }
14620                signal
14621            });
14622        let Some(signal) = matched else {
14623            continue;
14624        };
14625        let signal_codec = signal
14626            .get("payload_codec")
14627            .and_then(Value::as_str)
14628            .unwrap_or(&export_codec);
14629        let Some(payload) = event.payload.as_object_mut() else {
14630            continue;
14631        };
14632        if missing_payload(payload.get("arguments")) {
14633            if let Some(arguments) = signal
14634                .get("arguments")
14635                .filter(|value| !missing_payload(Some(value)))
14636            {
14637                let envelope = match arguments {
14638                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14639                    other => other.clone(),
14640                };
14641                payload.insert("arguments".to_string(), envelope);
14642            }
14643        }
14644        if payload
14645            .get("payload_codec")
14646            .and_then(Value::as_str)
14647            .unwrap_or_default()
14648            .is_empty()
14649        {
14650            payload.insert("payload_codec".to_string(), json!(signal_codec));
14651        }
14652    }
14653
14654    Ok(())
14655}
14656
14657fn missing_payload(value: Option<&Value>) -> bool {
14658    match value {
14659        None | Some(Value::Null) => true,
14660        Some(Value::String(value)) => value.is_empty(),
14661        Some(_) => false,
14662    }
14663}
14664
14665fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14666    let export_signals = task
14667        .history_export
14668        .as_ref()
14669        .and_then(|export| export.get("signals"))
14670        .and_then(Value::as_array)
14671        .cloned()
14672        .unwrap_or_default();
14673    let export_codec = task
14674        .history_export
14675        .as_ref()
14676        .and_then(|export| export.get("payloads"))
14677        .and_then(|payloads| payloads.get("codec"))
14678        .and_then(Value::as_str)
14679        .unwrap_or(&task.payload_codec);
14680    let mut name_offsets: HashMap<String, usize> = HashMap::new();
14681    let mut signals = Vec::new();
14682
14683    for event in &task.history_events {
14684        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14685            continue;
14686        }
14687
14688        let name = event
14689            .payload
14690            .get("signal_name")
14691            .and_then(Value::as_str)
14692            .unwrap_or_default();
14693        if name.is_empty() {
14694            continue;
14695        }
14696        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14697        let command_id = event
14698            .payload
14699            .get("workflow_command_id")
14700            .or_else(|| event.raw.get("workflow_command_id"))
14701            .and_then(Value::as_str);
14702        let matched_export = export_signals
14703            .iter()
14704            .find(|candidate| {
14705                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14706            })
14707            .or_else(|| {
14708                export_signals.iter().find(|candidate| {
14709                    command_id.is_some()
14710                        && candidate.get("command_id").and_then(Value::as_str) == command_id
14711                })
14712            })
14713            .or_else(|| {
14714                let offset = name_offsets.entry(name.to_string()).or_default();
14715                let candidate = export_signals
14716                    .iter()
14717                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14718                    .nth(*offset);
14719                if candidate.is_some() {
14720                    *offset += 1;
14721                }
14722                candidate
14723            });
14724        let codec = event
14725            .payload
14726            .get("payload_codec")
14727            .and_then(Value::as_str)
14728            .or_else(|| {
14729                matched_export
14730                    .and_then(|signal| signal.get("payload_codec"))
14731                    .and_then(Value::as_str)
14732            })
14733            .unwrap_or(export_codec);
14734        let raw_arguments = signal_history_payload(&event.payload)
14735            .filter(|value| !value.is_null())
14736            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14737        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14738        let workflow_sequence = event
14739            .payload
14740            .get("workflow_sequence")
14741            .and_then(value_as_u64)
14742            .or_else(|| {
14743                matched_export
14744                    .and_then(|signal| signal.get("workflow_sequence"))
14745                    .and_then(value_as_u64)
14746            });
14747
14748        signals.push(QuerySignal {
14749            id: signal_id.map(str::to_string).or_else(|| {
14750                matched_export
14751                    .and_then(|signal| signal.get("id"))
14752                    .and_then(Value::as_str)
14753                    .map(str::to_string)
14754            }),
14755            name: name.to_string(),
14756            arguments,
14757            avro_arguments,
14758            workflow_sequence,
14759        });
14760    }
14761
14762    if signals.is_empty() {
14763        for signal in export_signals {
14764            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14765                continue;
14766            }
14767            let Some(name) = signal.get("name").and_then(Value::as_str) else {
14768                continue;
14769            };
14770            let codec = signal
14771                .get("payload_codec")
14772                .and_then(Value::as_str)
14773                .unwrap_or(export_codec);
14774            let (arguments, avro_arguments) =
14775                decode_query_signal_arguments(signal.get("arguments"), codec)?;
14776            signals.push(QuerySignal {
14777                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14778                name: name.to_string(),
14779                arguments,
14780                avro_arguments,
14781                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14782            });
14783        }
14784        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14785    }
14786
14787    Ok(signals)
14788}
14789
14790fn decode_query_signal_arguments(
14791    raw: Option<&Value>,
14792    codec: &str,
14793) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14794    validate_payload_codec(codec)?;
14795    let decoded = match raw.filter(|value| !value.is_null()) {
14796        Some(value) => decode_wire_avro_value(value, codec)?,
14797        None => AvroValue::Array(Vec::new()),
14798    };
14799    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14800        unreachable!("normalize_avro_arguments always returns an array");
14801    };
14802    let arguments = avro_arguments
14803        .iter()
14804        .cloned()
14805        .map(AvroValue::into_json)
14806        .collect::<Result<Vec<_>>>()?;
14807    Ok((arguments, avro_arguments))
14808}
14809
14810fn value_as_u64(value: &Value) -> Option<u64> {
14811    value
14812        .as_u64()
14813        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14814}
14815
14816#[cfg(test)]
14817mod tests {
14818    use super::*;
14819    mod runtime_payloads;
14820    mod runtime_uploads;
14821    use std::{
14822        fs,
14823        io::{Read, Write},
14824        net::{SocketAddr, TcpListener, TcpStream},
14825        process::Command as ProcessCommand,
14826        sync::atomic::AtomicUsize,
14827        thread,
14828    };
14829
14830    #[derive(Clone, Copy, Debug)]
14831    enum InvalidTaskPayloadCodec {
14832        Missing,
14833        Null,
14834        NonString,
14835    }
14836
14837    impl InvalidTaskPayloadCodec {
14838        fn label(self) -> &'static str {
14839            match self {
14840                Self::Missing => "missing",
14841                Self::Null => "null",
14842                Self::NonString => "non-string",
14843            }
14844        }
14845
14846        fn apply(self, task: &mut Value) {
14847            let task = task.as_object_mut().expect("task fixture object");
14848            match self {
14849                Self::Missing => {
14850                    task.remove("payload_codec");
14851                }
14852                Self::Null => {
14853                    task.insert("payload_codec".to_string(), Value::Null);
14854                }
14855                Self::NonString => {
14856                    task.insert("payload_codec".to_string(), json!(42));
14857                }
14858            }
14859        }
14860    }
14861
14862    fn fixture_envelope(value: Value) -> Value {
14863        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14864    }
14865
14866    fn fixture_blob(value: Value) -> String {
14867        encode_payload(&value, DEFAULT_CODEC)
14868            .expect("encode Avro test fixture")
14869            .blob
14870    }
14871
14872    #[test]
14873    fn client_builder_rejects_the_sdk_owned_api_suffix() {
14874        for base_url in [
14875            "http://127.0.0.1:8080/api",
14876            "http://localhost:8080/api/",
14877            "https://runtime.example.test/namespaces/orders/api",
14878        ] {
14879            let error = Client::builder(base_url)
14880                .build()
14881                .expect_err("SDK-owned /api suffix must be rejected during build");
14882
14883            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14884            assert!(
14885                error.to_string().contains("SDK appends /api automatically"),
14886                "the validation error must explain how to fix the endpoint"
14887            );
14888        }
14889    }
14890
14891    #[test]
14892    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14893        for (base_url, expected) in [
14894            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14895            (
14896                "http://localhost:8080/durable-workflow/",
14897                "http://localhost:8080/durable-workflow",
14898            ),
14899            (
14900                "https://runtime.example.test/namespaces/orders",
14901                "https://runtime.example.test/namespaces/orders",
14902            ),
14903            (
14904                "https://runtime.example.test/gateway/api/namespaces/orders",
14905                "https://runtime.example.test/gateway/api/namespaces/orders",
14906            ),
14907            (
14908                "https://api.example.test/runtime/orders/",
14909                "https://api.example.test/runtime/orders",
14910            ),
14911        ] {
14912            let client = Client::builder(base_url)
14913                .build()
14914                .expect("Server and Cloud runtime base URL must remain valid");
14915
14916            assert_eq!(client.base_url, expected);
14917        }
14918    }
14919
14920    #[test]
14921    fn workflow_completion_uses_the_additive_command_protocol_floor() {
14922        assert_eq!(
14923            workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14924            WORKER_PROTOCOL_VERSION
14925        );
14926        assert_eq!(
14927            workflow_completion_protocol_version(&[json!({
14928                "type": "upsert_search_attributes",
14929                "attributes": {"OrderStatus": "waiting"},
14930            })]),
14931            SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14932        );
14933        assert_eq!(
14934            workflow_completion_protocol_version(&[json!({
14935                "type": "upsert_search_attributes",
14936                "attributes": {"OrderStatus": "waiting"},
14937                "attribute_types": {"OrderStatus": "keyword"},
14938            })]),
14939            TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14940        );
14941        assert_eq!(
14942            workflow_completion_protocol_version(&[
14943                json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14944                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14945            ]),
14946            MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14947        );
14948        assert_eq!(
14949            workflow_completion_protocol_version(&[
14950                json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14951                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14952            ]),
14953            CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14954        );
14955        assert_eq!(
14956            workflow_completion_protocol_version(&[json!({
14957                "type": "open_condition_wait",
14958                "condition_wait_occurrence_id": "rust:condition-wait:0",
14959                "condition_key": "ready",
14960            })]),
14961            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14962        );
14963        assert_eq!(
14964            workflow_completion_protocol_version_with_message_streams(
14965                &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14966                true,
14967            ),
14968            MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14969        );
14970        assert_eq!(
14971            workflow_completion_protocol_version_with_message_streams(
14972                &[json!({
14973                    "type": "open_condition_wait",
14974                    "condition_wait_occurrence_id": "rust:condition-wait:0",
14975                    "condition_key": "ready",
14976                })],
14977                true,
14978            ),
14979            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14980        );
14981    }
14982
14983    #[test]
14984    fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14985        let manifest = portable_worker_affinity_capability_manifest();
14986
14987        for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14988            assert_eq!(manifest[capability]["supported"], json!(false));
14989            assert_eq!(
14990                manifest[capability]["minimum_protocol_version"],
14991                json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14992            );
14993            assert!(manifest[capability]["reason"]
14994                .as_str()
14995                .is_some_and(|reason| !reason.is_empty()));
14996        }
14997    }
14998
14999    fn typed_fidelity_probe() -> AvroValue {
15000        AvroValue::Map(BTreeMap::from([
15001            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
15002            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
15003            (
15004                "numeric".to_string(),
15005                AvroValue::Map(BTreeMap::from([
15006                    ("0".to_string(), AvroValue::String("zero".to_string())),
15007                    ("1".to_string(), AvroValue::String("one".to_string())),
15008                ])),
15009            ),
15010            (
15011                "nested".to_string(),
15012                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
15013                    "enabled".to_string(),
15014                    AvroValue::Boolean(true),
15015                )]))]),
15016            ),
15017            (
15018                "projection_collisions".to_string(),
15019                AvroValue::Array(projection_collision_probe()),
15020            ),
15021        ]))
15022    }
15023
15024    fn projection_collision_probe() -> Vec<AvroValue> {
15025        vec![
15026            AvroValue::Map(BTreeMap::from([
15027                ("$type".to_string(), AvroValue::String("bytes".to_string())),
15028                (
15029                    "base64".to_string(),
15030                    AvroValue::String("ordinary user text".to_string()),
15031                ),
15032            ])),
15033            AvroValue::Map(BTreeMap::from([
15034                ("$type".to_string(), AvroValue::String("map".to_string())),
15035                (
15036                    "entries".to_string(),
15037                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
15038                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
15039                        (
15040                            "value".to_string(),
15041                            AvroValue::String("user map".to_string()),
15042                        ),
15043                    ]))]),
15044                ),
15045            ])),
15046        ]
15047    }
15048
15049    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15050    struct TypedContract {
15051        nested: TypedNested,
15052        mode: TypedMode,
15053        optional: Option<String>,
15054        absent: Option<String>,
15055        items: Vec<i64>,
15056        labels: BTreeMap<String, String>,
15057        bytes: serde_bytes::ByteBuf,
15058        signed: i64,
15059        finite: f64,
15060    }
15061
15062    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15063    struct TypedNested {
15064        enabled: bool,
15065    }
15066
15067    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15068    enum TypedMode {
15069        Detailed { label: String },
15070    }
15071
15072    fn typed_contract() -> TypedContract {
15073        TypedContract {
15074            nested: TypedNested { enabled: true },
15075            mode: TypedMode::Detailed {
15076                label: "compiler-checked".to_string(),
15077            },
15078            optional: Some("present".to_string()),
15079            absent: None,
15080            items: vec![i64::MIN, 0, i64::MAX],
15081            labels: BTreeMap::from([
15082                ("language".to_string(), "rust".to_string()),
15083                ("wire".to_string(), "avro".to_string()),
15084            ]),
15085            bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
15086            signed: -9_223_372_036_854_775_000,
15087            finite: 12.5,
15088        }
15089    }
15090
15091    #[derive(Clone, Debug, Default, PartialEq)]
15092    struct ReplayCounterState {
15093        loaded: Option<String>,
15094        count: i64,
15095        finished: bool,
15096    }
15097
15098    fn replay_counter_worker() -> Worker {
15099        let client = Client::new("http://127.0.0.1:8080").expect("client");
15100        let mut worker = Worker::new(client, "rust-workers");
15101        worker.register_replayed_workflow(
15102            "replay-counter",
15103            ReplayCounterState::default,
15104            |ctx, _input, state| async move {
15105                let loaded = ctx.activity("load-counter", json!([])).await?;
15106                state.update(|current| {
15107                    current.loaded = loaded.as_str().map(str::to_string);
15108                })?;
15109                for _ in 0..2 {
15110                    let signal = ctx.wait_signal("increment").await?;
15111                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15112                    state.update(|current| current.count += amount)?;
15113                }
15114                state.update(|current| current.finished = true)?;
15115                state.read(|current| Ok(json!(current.count)))?
15116            },
15117        );
15118        worker.register_replayed_query::<ReplayCounterState, _, _>(
15119            "replay-counter",
15120            "current",
15121            |_ctx, state, _args| async move {
15122                Ok(json!({
15123                    "loaded": state.loaded,
15124                    "count": state.count,
15125                    "finished": state.finished,
15126                }))
15127            },
15128        );
15129        worker.register_replayed_query::<ReplayCounterState, _, _>(
15130            "replay-counter",
15131            "detached-mutation",
15132            |_ctx, state, _args| async move {
15133                let mut detached = (*state).clone();
15134                detached.count = 999;
15135                Ok(json!(detached.count))
15136            },
15137        );
15138        worker.register_replayed_query::<ReplayCounterState, _, _>(
15139            "replay-counter",
15140            "failed-mutation",
15141            |_ctx, state, _args| async move {
15142                let mut detached = (*state).clone();
15143                detached.count = 999;
15144                Err(Error::WorkerLoop("query refused".to_string()))
15145            },
15146        );
15147        worker
15148    }
15149
15150    fn replay_counter_query(
15151        query_name: &str,
15152        history_events: Value,
15153        run_status: &str,
15154    ) -> QueryTask {
15155        let arguments = fixture_envelope(json!([]));
15156        serde_json::from_value(json!({
15157            "query_task_id": format!("query-{query_name}"),
15158            "workflow_type": "replay-counter",
15159            "query_name": query_name,
15160            "payload_codec": DEFAULT_CODEC,
15161            "workflow_arguments": arguments.clone(),
15162            "query_arguments": arguments,
15163            "history_events": history_events,
15164            "run_status": run_status,
15165        }))
15166        .expect("query task")
15167    }
15168
15169    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15170        workflow_context_with_codec(history, DEFAULT_CODEC)
15171    }
15172
15173    fn workflow_context_with_codec(
15174        history: Vec<HistoryEvent>,
15175        payload_codec: &str,
15176    ) -> WorkflowContext {
15177        WorkflowContext {
15178            state: Arc::new(Mutex::new(
15179                WorkflowState::new_with_identity(
15180                    history,
15181                    None,
15182                    None,
15183                    "rust-workers".to_string(),
15184                    payload_codec.to_string(),
15185                    None,
15186                )
15187                .expect("valid workflow history"),
15188            )),
15189        }
15190    }
15191
15192    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15193        HistoryEvent {
15194            event_type: event_type.to_string(),
15195            payload,
15196            raw: HashMap::new(),
15197        }
15198    }
15199
15200    fn parallel_path_entry(
15201        kind: &str,
15202        base: u64,
15203        size: usize,
15204        index: usize,
15205    ) -> ParallelGroupMetadata {
15206        parallel_group_entry(base, size, index, kind)
15207    }
15208
15209    fn parallel_history_event(
15210        event_type: &str,
15211        sequence: u64,
15212        identity_field: &str,
15213        identity: &str,
15214        path: Vec<ParallelGroupMetadata>,
15215        result: Option<Value>,
15216    ) -> HistoryEvent {
15217        let mut payload = serde_json::Map::from_iter([
15218            ("sequence".to_string(), json!(sequence)),
15219            (identity_field.to_string(), json!(identity)),
15220        ]);
15221        let inner = path.last().expect("parallel history path");
15222        apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15223        payload.insert("parallel_group_path".to_string(), json!(path));
15224        if let Some(result) = result {
15225            let field = if event_type == "ChildRunCompleted" {
15226                "result"
15227            } else {
15228                "result"
15229            };
15230            payload.insert(field.to_string(), fixture_envelope(result));
15231            payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15232        }
15233        history_event(event_type, Value::Object(payload))
15234    }
15235
15236    fn nested_parallel_operations() -> Vec<ParallelOperation> {
15237        vec![
15238            ParallelOperation::activity("first", json!([])),
15239            ParallelOperation::group(vec![
15240                ParallelOperation::child_workflow(
15241                    "second",
15242                    ChildWorkflowOptions::new("child-workers"),
15243                    json!([]),
15244                ),
15245                ParallelOperation::activity("third", json!([])),
15246            ]),
15247        ]
15248    }
15249
15250    fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15251        let outer = [
15252            parallel_path_entry("mixed", 1, 3, 0),
15253            parallel_path_entry("mixed", 1, 3, 1),
15254            parallel_path_entry("mixed", 1, 3, 2),
15255        ];
15256        [
15257            vec![outer[0].clone()],
15258            vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15259            vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15260        ]
15261    }
15262
15263    #[test]
15264    fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15265        let ctx = workflow_context(Vec::new());
15266        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15267        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15268
15269        assert!(matches!(
15270            call.as_mut().poll(&mut task_context),
15271            Poll::Pending
15272        ));
15273        let commands = ctx.take_commands().expect("parallel commands");
15274        assert_eq!(
15275            commands
15276                .iter()
15277                .map(|command| command["type"].as_str().unwrap_or_default())
15278                .collect::<Vec<_>>(),
15279            [
15280                "schedule_activity",
15281                "start_child_workflow",
15282                "schedule_activity"
15283            ]
15284        );
15285        let paths = nested_parallel_paths();
15286        for (command, path) in commands.iter().zip(paths) {
15287            assert_eq!(command["parallel_group_path"], json!(path));
15288            assert_eq!(
15289                command["parallel_group_id"],
15290                json!(path.last().expect("inner group").parallel_group_id)
15291            );
15292        }
15293    }
15294
15295    fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15296        let paths = nested_parallel_paths();
15297        let third = parallel_history_event(
15298            "ActivityCompleted",
15299            3,
15300            "activity_type",
15301            "third",
15302            paths[2].clone(),
15303            Some(json!("three")),
15304        );
15305        vec![
15306            parallel_history_event(
15307                "ActivityCompleted",
15308                1,
15309                "activity_type",
15310                "first",
15311                paths[0].clone(),
15312                Some(json!("one")),
15313            ),
15314            parallel_history_event(
15315                "ChildWorkflowScheduled",
15316                2,
15317                "child_workflow_type",
15318                "second",
15319                paths[1].clone(),
15320                None,
15321            ),
15322            parallel_history_event(
15323                "ChildRunCompleted",
15324                2,
15325                "child_workflow_type",
15326                "second",
15327                paths[1].clone(),
15328                Some(json!("two")),
15329            ),
15330            third.clone(),
15331            third,
15332        ]
15333    }
15334
15335    #[test]
15336    fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15337        for _restart_or_completed_replay in 0..2 {
15338            let ctx = workflow_context(completed_nested_parallel_history());
15339            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15340            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15341            let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15342                panic!("completed nested parallel history must replay");
15343            };
15344            assert_eq!(
15345                results,
15346                vec![
15347                    ParallelResult::Activity(json!("one")),
15348                    ParallelResult::Group(vec![
15349                        ParallelResult::ChildWorkflow(ChildWorkflowResult {
15350                            parent: WorkflowIdentity {
15351                                workflow_id: None,
15352                                run_id: None,
15353                            },
15354                            child: WorkflowIdentity {
15355                                workflow_id: None,
15356                                run_id: None,
15357                            },
15358                            child_workflow_type: Some("second".to_string()),
15359                            result: json!("two"),
15360                        }),
15361                        ParallelResult::Activity(json!("three")),
15362                    ]),
15363                ]
15364            );
15365            assert!(ctx.take_commands().expect("commands").is_empty());
15366            ctx.ensure_history_consumed().expect("history consumed");
15367        }
15368    }
15369
15370    #[test]
15371    fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15372        let paths = nested_parallel_paths();
15373        let history = vec![
15374            parallel_history_event(
15375                "ActivityCompleted",
15376                1,
15377                "activity_type",
15378                "first",
15379                paths[0].clone(),
15380                Some(json!("one")),
15381            ),
15382            parallel_history_event(
15383                "ChildWorkflowScheduled",
15384                2,
15385                "child_workflow_type",
15386                "second",
15387                paths[1].clone(),
15388                None,
15389            ),
15390            parallel_history_event(
15391                "ChildRunFailed",
15392                2,
15393                "child_workflow_type",
15394                "second",
15395                paths[1].clone(),
15396                None,
15397            ),
15398            parallel_history_event(
15399                "ActivityCompleted",
15400                3,
15401                "activity_type",
15402                "third",
15403                paths[2].clone(),
15404                Some(json!("late")),
15405            ),
15406        ];
15407        let ctx = workflow_context(history);
15408        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15409        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15410        let outcome = call.as_mut().poll(&mut task_context);
15411        let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15412            panic!("one failed child must return a typed partial failure: {outcome:?}");
15413        };
15414        assert_eq!(failure.member_path, [1, 0]);
15415        assert_eq!(failure.group_id, "parallel-calls:1:3");
15416        assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15417        assert_eq!(
15418            failure
15419                .completed
15420                .iter()
15421                .map(|completion| completion.member_path.clone())
15422                .collect::<Vec<_>>(),
15423            [vec![0], vec![1, 1]]
15424        );
15425    }
15426
15427    #[test]
15428    fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15429        let paths = nested_parallel_paths();
15430        let history = vec![
15431            parallel_history_event(
15432                "ActivityScheduled",
15433                1,
15434                "activity_type",
15435                "first",
15436                paths[0].clone(),
15437                None,
15438            ),
15439            parallel_history_event(
15440                "ChildWorkflowScheduled",
15441                2,
15442                "child_workflow_type",
15443                "second",
15444                paths[1].clone(),
15445                None,
15446            ),
15447            parallel_history_event(
15448                "ActivityScheduled",
15449                3,
15450                "activity_type",
15451                "third",
15452                paths[2].clone(),
15453                None,
15454            ),
15455        ];
15456        for _restart in 0..2 {
15457            let ctx = workflow_context(history.clone());
15458            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15459            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15460            let outcome = call.as_mut().poll(&mut task_context);
15461            assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15462            assert!(ctx.take_commands().expect("commands").is_empty());
15463        }
15464    }
15465
15466    fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15467        vec![selection_group_entry(
15468            1,
15469            2,
15470            index,
15471            "activity",
15472            &SelectionMemberMetadata {
15473                key: SelectionKey::Name(key.to_string()),
15474                index,
15475                base_sequence: index as u64 + 1,
15476                size: 1,
15477                kind: "activity".to_string(),
15478            },
15479        )]
15480    }
15481
15482    fn selection_activity_event(
15483        event_type: &str,
15484        index: usize,
15485        key: &str,
15486        result: Option<Value>,
15487    ) -> HistoryEvent {
15488        let sequence = index as u64 + 1;
15489        let mut event = parallel_history_event(
15490            event_type,
15491            sequence,
15492            "activity_type",
15493            &format!("{key}-activity"),
15494            selection_path(index, key),
15495            result,
15496        );
15497        event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15498        event.raw.insert(
15499            "id".to_string(),
15500            json!(if event_type == "ActivityCompleted" {
15501                format!("event-{key}")
15502            } else {
15503                format!("{event_type}-{key}")
15504            }),
15505        );
15506        event
15507    }
15508
15509    fn selection_winner_marker() -> HistoryEvent {
15510        history_event(
15511            "SelectionResolved",
15512            json!({
15513                "selection_group_id": "select-calls:1:2",
15514                "selection_group_base_sequence": 1,
15515                "selection_group_size": 2,
15516                "member_key": "fast",
15517                "member_index": 1,
15518                "member_base_sequence": 2,
15519                "member_size": 1,
15520                "operation_kind": "activity",
15521                "operation_identity": "activity-fast",
15522                "outcome": "completed",
15523                "resolution_event_id": "event-fast",
15524                "resolution_event_type": "ActivityCompleted",
15525            }),
15526        )
15527    }
15528
15529    fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15530        ctx.select_keyed(vec![
15531            (
15532                "slow",
15533                ParallelOperation::activity_with_options(
15534                    "slow-activity",
15535                    ActivityOptions::new().task_queue("default"),
15536                    json!([]),
15537                ),
15538            ),
15539            (
15540                "fast",
15541                ParallelOperation::activity_with_options(
15542                    "fast-activity",
15543                    ActivityOptions::new().task_queue("default"),
15544                    json!([]),
15545                ),
15546            ),
15547        ])
15548    }
15549
15550    fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15551        let ctx = workflow_context(history);
15552        let mut call = Box::pin(keyed_activity_selection(&ctx));
15553        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15554        let selected = match call.as_mut().poll(&mut task_context) {
15555            Poll::Ready(Ok(selected)) => selected,
15556            Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15557            Poll::Pending => panic!("persisted selection winner must replay without pending"),
15558        };
15559        assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15560        assert_eq!(
15561            selected.value,
15562            Some(ParallelResult::Activity(json!("winner-value")))
15563        );
15564        let slow = selected
15565            .handle(&SelectionKey::Name("slow".to_string()))
15566            .expect("slow handle")
15567            .clone();
15568        let mut await_slow = Box::pin(slow.await_result());
15569        assert!(matches!(
15570            await_slow.as_mut().poll(&mut task_context),
15571            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15572        ));
15573        assert!(ctx.take_commands().expect("commands").is_empty());
15574    }
15575
15576    const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15577
15578    fn canonical_selection_history() -> Vec<HistoryEvent> {
15579        const FIXTURE: &[u8] =
15580            include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15581        assert_eq!(
15582            format!("{:x}", Sha256::digest(FIXTURE)),
15583            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15584        );
15585        let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15586
15587        serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15588    }
15589
15590    #[test]
15591    fn selection_fresh_process_entrypoint() {
15592        let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15593            return;
15594        };
15595        let persisted = fs::read(path).expect("persisted selection history");
15596        assert_eq!(
15597            format!("{:x}", Sha256::digest(&persisted)),
15598            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15599        );
15600        let fixture: Value =
15601            serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15602        let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15603            .expect("valid persisted selection history");
15604
15605        assert_persisted_selection_replay(history);
15606    }
15607
15608    #[test]
15609    fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15610        let ctx = workflow_context(Vec::new());
15611        let mut call = Box::pin(keyed_activity_selection(&ctx));
15612        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15613
15614        assert!(matches!(
15615            call.as_mut().poll(&mut task_context),
15616            Poll::Pending
15617        ));
15618        let commands = ctx.take_commands().expect("selection commands");
15619        assert_eq!(commands.len(), 2);
15620        assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15621        assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15622        assert!(commands.iter().all(|command| {
15623            command["parallel_group_id"] == json!("select-calls:1:2")
15624                && command["parallel_group_mode"] == json!("select")
15625        }));
15626    }
15627
15628    #[test]
15629    fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15630        let ctx = workflow_context(Vec::new());
15631        let mut invalid = Box::pin(ctx.select_keyed(vec![(
15632            "",
15633            ParallelOperation::activity("invalid", json!([])),
15634        )]));
15635        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15636        assert!(matches!(
15637            invalid.as_mut().poll(&mut task_context),
15638            Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15639                reason: "selection_key_invalid",
15640                ..
15641            })))
15642        ));
15643
15644        for invalid_key in [json!(""), json!(-1)] {
15645            let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15646            event.payload["selection_member_key"] = invalid_key.clone();
15647            event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15648            assert!(matches!(
15649                WorkflowState::new_with_identity(
15650                    vec![event],
15651                    None,
15652                    None,
15653                    "rust-workers".to_string(),
15654                    DEFAULT_CODEC.to_string(),
15655                    None,
15656                ),
15657                Err(Error::NonDeterministicReplay(_))
15658            ));
15659        }
15660    }
15661
15662    #[test]
15663    fn selection_preserves_valid_named_and_numeric_keys() {
15664        let ctx = workflow_context(Vec::new());
15665        let mut selection = Box::pin(ctx.select_keyed(vec![
15666            (
15667                SelectionKey::Index(0),
15668                ParallelOperation::activity("numeric", json!([])),
15669            ),
15670            (
15671                SelectionKey::Name("named".to_string()),
15672                ParallelOperation::timer(Duration::from_secs(1)),
15673            ),
15674        ]));
15675        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15676
15677        assert!(matches!(
15678            selection.as_mut().poll(&mut task_context),
15679            Poll::Pending
15680        ));
15681        let commands = ctx.take_commands().expect("selection commands");
15682        assert_eq!(commands[0]["selection_member_key"], json!(0));
15683        assert_eq!(commands[1]["selection_member_key"], json!("named"));
15684    }
15685
15686    #[test]
15687    fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15688        let history = canonical_selection_history();
15689        assert_persisted_selection_replay(history.clone());
15690
15691        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15692            .join("tests/fixtures/durable_selection_runtime_history.json");
15693        let output =
15694            ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15695                .args([
15696                    "--exact",
15697                    "tests::selection_fresh_process_entrypoint",
15698                    "--nocapture",
15699                ])
15700                .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15701                .output()
15702                .expect("run fresh selection replay process");
15703
15704        assert!(
15705            output.status.success(),
15706            "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15707            String::from_utf8_lossy(&output.stdout),
15708            String::from_utf8_lossy(&output.stderr),
15709        );
15710    }
15711
15712    #[test]
15713    fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15714        let mut history = canonical_selection_history();
15715        history.retain(|event| event.event_type != "SelectionResolved");
15716        let ctx = workflow_context(history);
15717        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15718        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15719
15720        assert!(matches!(
15721            selection.as_mut().poll(&mut task_context),
15722            Poll::Pending
15723        ));
15724        assert!(ctx.take_commands().expect("commands").is_empty());
15725        assert!(
15726            ctx.matched_recorded_pending()
15727                .expect("selection pending state"),
15728            "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15729        );
15730    }
15731
15732    #[test]
15733    fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15734        for (terminal_event, predicate_satisfied, timeout_seconds) in [
15735            ("ConditionWaitSatisfied", true, None),
15736            ("ConditionWaitTimedOut", false, Some(0)),
15737        ] {
15738            let member = SelectionMemberMetadata {
15739                key: SelectionKey::Name("condition".to_string()),
15740                index: 0,
15741                base_sequence: 1,
15742                size: 1,
15743                kind: "condition".to_string(),
15744            };
15745            let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15746            let mut payload = json!({
15747                "sequence": 1,
15748                "condition_wait_id": "condition-1",
15749                "condition_wait_occurrence_id": "rust:condition-wait:0",
15750                "condition_key": "ready",
15751                "condition_definition_fingerprint": "sha256:ready-v1",
15752                "parallel_group_path": path,
15753            });
15754            payload
15755                .as_object_mut()
15756                .expect("condition history payload")
15757                .extend(
15758                    serde_json::to_value(&path[0])
15759                        .expect("condition selection metadata")
15760                        .as_object()
15761                        .expect("condition selection metadata object")
15762                        .clone(),
15763                );
15764            if let Some(timeout_seconds) = timeout_seconds {
15765                payload["timeout_seconds"] = json!(timeout_seconds);
15766            }
15767            let history = vec![
15768                history_event("ConditionWaitOpened", payload.clone()),
15769                history_event(terminal_event, payload),
15770            ];
15771            let ctx = workflow_context(history);
15772            let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15773            if timeout_seconds.is_some() {
15774                options = options.timeout(Duration::ZERO);
15775            }
15776            let mut selection = Box::pin(ctx.select_keyed(vec![(
15777                "condition",
15778                ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15779            )]));
15780            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15781
15782            assert!(matches!(
15783                selection.as_mut().poll(&mut task_context),
15784                Poll::Pending
15785            ));
15786            assert!(ctx.take_commands().expect("commands").is_empty());
15787            assert!(
15788                ctx.matched_recorded_pending()
15789                    .expect("condition selection pending state"),
15790                "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15791            );
15792        }
15793    }
15794
15795    #[test]
15796    fn selection_immediate_condition_members_open_a_durable_wait() {
15797        for predicate_satisfied in [true, false] {
15798            let ctx = workflow_context(Vec::new());
15799            let mut selection = Box::pin(ctx.select_keyed(vec![(
15800                "condition",
15801                ParallelOperation::condition(
15802                    ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15803                    move || Ok(predicate_satisfied),
15804                ),
15805            )]));
15806            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15807
15808            assert!(matches!(
15809                selection.as_mut().poll(&mut task_context),
15810                Poll::Pending
15811            ));
15812            let commands = ctx.take_commands().expect("condition selection command");
15813            assert_eq!(commands.len(), 1);
15814            assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15815            assert_eq!(commands[0]["timeout_seconds"], json!(0));
15816            assert_eq!(
15817                commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15818                json!("select")
15819            );
15820        }
15821    }
15822
15823    #[test]
15824    fn selection_loser_cancellation_is_explicit_and_idempotent() {
15825        let history = vec![
15826            selection_activity_event("ActivityScheduled", 0, "slow", None),
15827            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15828            selection_winner_marker(),
15829        ];
15830        let ctx = workflow_context(history.clone());
15831        let mut call = Box::pin(keyed_activity_selection(&ctx));
15832        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15833        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15834            panic!("winner must replay");
15835        };
15836        let slow = selected
15837            .handle(&SelectionKey::Name("slow".to_string()))
15838            .expect("slow handle")
15839            .clone();
15840        let mut cancel = Box::pin(slow.cancel());
15841        assert!(matches!(
15842            cancel.as_mut().poll(&mut task_context),
15843            Poll::Pending
15844        ));
15845        assert!(matches!(
15846            cancel.as_mut().poll(&mut task_context),
15847            Poll::Pending
15848        ));
15849        let commands = ctx.take_commands().expect("cancel command");
15850        assert_eq!(commands.len(), 1);
15851        assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15852        assert_eq!(commands[0]["member_key"], json!("slow"));
15853
15854        let mut cancelled_history = history;
15855        cancelled_history.push(history_event(
15856            "SelectionOperationCancelled",
15857            json!({
15858                "selection_group_id": "select-calls:1:2",
15859                "member_key": "slow",
15860                "member_index": 0,
15861                "member_base_sequence": 1,
15862                "member_size": 1,
15863                "operation_kind": "activity",
15864                "operation_identity": "activity-slow",
15865                "cancelled_at": "2026-08-27T00:00:00Z",
15866            }),
15867        ));
15868        let replayed = workflow_context(cancelled_history);
15869        let mut call = Box::pin(keyed_activity_selection(&replayed));
15870        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15871            panic!("winner must replay after cancellation");
15872        };
15873        let slow = selected
15874            .handle(&SelectionKey::Name("slow".to_string()))
15875            .expect("slow handle")
15876            .clone();
15877        let mut cancel = Box::pin(slow.cancel());
15878        assert!(matches!(
15879            cancel.as_mut().poll(&mut task_context),
15880            Poll::Ready(Ok(()))
15881        ));
15882        assert!(replayed.take_commands().expect("commands").is_empty());
15883    }
15884
15885    #[test]
15886    fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15887        let base_history = vec![
15888            selection_activity_event("ActivityScheduled", 0, "slow", None),
15889            selection_activity_event("ActivityScheduled", 1, "fast", None),
15890            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15891            selection_winner_marker(),
15892        ];
15893        for (field, corrupt) in [
15894            ("member_key", json!("fast")),
15895            ("member_index", json!(1)),
15896            ("member_base_sequence", json!(3)),
15897            ("member_size", json!(2)),
15898            ("operation_kind", json!("timer")),
15899            ("operation_identity", json!("forged")),
15900        ] {
15901            let mut cancellation = json!({
15902                "selection_group_id": "select-calls:1:2",
15903                "member_key": "slow",
15904                "member_index": 0,
15905                "member_base_sequence": 1,
15906                "member_size": 1,
15907                "operation_kind": "activity",
15908                "operation_identity": "activity-slow",
15909            });
15910            cancellation[field] = corrupt;
15911            let mut history = base_history.clone();
15912            history.push(history_event("SelectionOperationCancelled", cancellation));
15913            let ctx = workflow_context(history);
15914            let mut selection = Box::pin(keyed_activity_selection(&ctx));
15915            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15916
15917            assert!(matches!(
15918                selection.as_mut().poll(&mut task_context),
15919                Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15920            ));
15921        }
15922    }
15923
15924    #[test]
15925    fn selection_child_identity_prefers_the_durable_run_id() {
15926        let ctx = workflow_context(vec![history_event(
15927            "ChildWorkflowScheduled",
15928            json!({
15929                "sequence": 1,
15930                "child_workflow_type": "child",
15931                "child_workflow_instance_id": "child-instance",
15932                "child_workflow_run_id": "child-run",
15933            }),
15934        )]);
15935        let state = ctx.state.lock().expect("workflow state");
15936
15937        assert_eq!(
15938            selection_operation_identity(&state, "child", 1, 1),
15939            "child-run"
15940        );
15941    }
15942
15943    #[test]
15944    fn selection_activity_identity_requires_canonical_execution_id() {
15945        let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15946        let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15947        let mut fast_completed =
15948            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15949        for event in [&mut fast_open, &mut fast_completed] {
15950            event
15951                .payload
15952                .as_object_mut()
15953                .expect("activity payload")
15954                .remove("activity_execution_id");
15955            event.payload["activity_id"] = json!("forged-activity-id");
15956        }
15957        let mut marker = selection_winner_marker();
15958        marker.payload["operation_identity"] = json!("forged-activity-id");
15959        let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15960        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15961        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15962
15963        assert!(matches!(
15964            selection.as_mut().poll(&mut task_context),
15965            Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15966        ));
15967    }
15968
15969    #[test]
15970    fn selection_completion_before_cancellation_remains_awaitable() {
15971        let history = vec![
15972            selection_activity_event("ActivityScheduled", 0, "slow", None),
15973            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15974            selection_winner_marker(),
15975            selection_activity_event(
15976                "ActivityCompleted",
15977                0,
15978                "slow",
15979                Some(json!("completed-first")),
15980            ),
15981        ];
15982        let ctx = workflow_context(history);
15983        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15984        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15985        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15986            panic!("winner must replay");
15987        };
15988        let slow = selected
15989            .handle(&SelectionKey::Name("slow".to_string()))
15990            .expect("slow handle")
15991            .clone();
15992        let mut cancel = Box::pin(slow.cancel());
15993        assert!(matches!(
15994            cancel.as_mut().poll(&mut task_context),
15995            Poll::Ready(Ok(()))
15996        ));
15997        let mut await_slow = Box::pin(slow.await_result());
15998        assert!(matches!(
15999            await_slow.as_mut().poll(&mut task_context),
16000            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
16001        ));
16002        let commands = ctx.take_commands().expect("commands");
16003        assert!(commands.is_empty());
16004    }
16005
16006    #[test]
16007    fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
16008        let nested_member = SelectionMemberMetadata {
16009            key: SelectionKey::Name("nested".to_string()),
16010            index: 0,
16011            base_sequence: 1,
16012            size: 2,
16013            kind: "group".to_string(),
16014        };
16015        let deadline_member = SelectionMemberMetadata {
16016            key: SelectionKey::Name("deadline".to_string()),
16017            index: 1,
16018            base_sequence: 3,
16019            size: 1,
16020            kind: "timer".to_string(),
16021        };
16022        let nested_paths = [
16023            vec![
16024                selection_group_entry(1, 3, 0, "mixed", &nested_member),
16025                parallel_group_entry(1, 2, 0, "activity"),
16026            ],
16027            vec![
16028                selection_group_entry(1, 3, 1, "mixed", &nested_member),
16029                parallel_group_entry(1, 2, 1, "activity"),
16030            ],
16031        ];
16032        let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
16033        let mut timer_fired = parallel_history_event(
16034            "TimerFired",
16035            3,
16036            "timer_id",
16037            "timer-3",
16038            deadline_path.clone(),
16039            None,
16040        );
16041        timer_fired.payload["delay_seconds"] = json!(0);
16042        timer_fired
16043            .raw
16044            .insert("id".to_string(), json!("timer-fired"));
16045        let mut timer_scheduled = parallel_history_event(
16046            "TimerScheduled",
16047            3,
16048            "timer_id",
16049            "timer-3",
16050            deadline_path,
16051            None,
16052        );
16053        timer_scheduled.payload["delay_seconds"] = json!(0);
16054        let history = vec![
16055            parallel_history_event(
16056                "ActivityScheduled",
16057                1,
16058                "activity_type",
16059                "nested-first",
16060                nested_paths[0].clone(),
16061                None,
16062            ),
16063            parallel_history_event(
16064                "ActivityScheduled",
16065                2,
16066                "activity_type",
16067                "nested-second",
16068                nested_paths[1].clone(),
16069                None,
16070            ),
16071            timer_scheduled,
16072            timer_fired,
16073            history_event(
16074                "SelectionResolved",
16075                json!({
16076                    "selection_group_id": "select-calls:1:3",
16077                    "selection_group_base_sequence": 1,
16078                    "selection_group_size": 3,
16079                    "member_key": "deadline",
16080                    "member_index": 1,
16081                    "member_base_sequence": 3,
16082                    "member_size": 1,
16083                    "operation_kind": "timer",
16084                    "operation_identity": "timer-3",
16085                    "outcome": "completed",
16086                    "resolution_event_id": "timer-fired",
16087                    "resolution_event_type": "TimerFired",
16088                }),
16089            ),
16090            parallel_history_event(
16091                "ActivityFailed",
16092                2,
16093                "activity_type",
16094                "nested-second",
16095                nested_paths[1].clone(),
16096                None,
16097            ),
16098        ];
16099        let ctx = workflow_context(history);
16100        let mut selection = Box::pin(ctx.select_keyed(vec![
16101            (
16102                "nested",
16103                ParallelOperation::group(vec![
16104                    ParallelOperation::activity("nested-first", json!([])),
16105                    ParallelOperation::activity("nested-second", json!([])),
16106                ]),
16107            ),
16108            ("deadline", ParallelOperation::timer(Duration::ZERO)),
16109        ]));
16110        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16111        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16112            panic!("deadline winner must replay");
16113        };
16114        let nested = selected
16115            .handle(&SelectionKey::Name("nested".to_string()))
16116            .expect("nested handle")
16117            .clone();
16118        let mut cancel = Box::pin(nested.cancel());
16119        assert!(matches!(
16120            cancel.as_mut().poll(&mut task_context),
16121            Poll::Ready(Ok(()))
16122        ));
16123        let mut await_nested = Box::pin(nested.await_result());
16124
16125        assert!(matches!(
16126            await_nested.as_mut().poll(&mut task_context),
16127            Poll::Ready(Err(Error::ActivityFailed(_)))
16128        ));
16129        assert!(ctx.take_commands().expect("commands").is_empty());
16130    }
16131
16132    #[test]
16133    fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16134        let ctx = workflow_context(Vec::new());
16135        let mut call = Box::pin(ctx.select(vec![
16136            ParallelOperation::child_workflow(
16137                "child",
16138                ChildWorkflowOptions::new("children"),
16139                json!([]),
16140            ),
16141            ParallelOperation::timer(Duration::from_secs(30)),
16142            ParallelOperation::signal("approval"),
16143            ParallelOperation::condition(
16144                ConditionWaitOptions::new("ready", "sha256:ready"),
16145                || Ok(false),
16146            ),
16147            ParallelOperation::group(vec![
16148                ParallelOperation::activity("nested-one", json!([])),
16149                ParallelOperation::activity("nested-two", json!([])),
16150            ]),
16151        ]));
16152        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16153        assert!(matches!(
16154            call.as_mut().poll(&mut task_context),
16155            Poll::Pending
16156        ));
16157        let commands = ctx.take_commands().expect("selection commands");
16158        assert_eq!(
16159            commands
16160                .iter()
16161                .map(|command| command["type"].as_str().unwrap_or_default())
16162                .collect::<Vec<_>>(),
16163            [
16164                "start_child_workflow",
16165                "start_timer",
16166                "open_signal_wait",
16167                "open_condition_wait",
16168                "schedule_activity",
16169                "schedule_activity",
16170            ]
16171        );
16172        assert!(commands.iter().all(|command| {
16173            command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16174        }));
16175        assert_eq!(
16176            commands[4]["parallel_group_path"].as_array().map(Vec::len),
16177            Some(2)
16178        );
16179        assert_eq!(
16180            commands[4]["parallel_group_path"][0]["selection_member_kind"],
16181            json!("group")
16182        );
16183        assert_eq!(
16184            commands[5]["parallel_group_path"][0]["selection_member_kind"],
16185            json!("group")
16186        );
16187
16188        let one_leaf_ctx = workflow_context(Vec::new());
16189        let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16190            ParallelOperation::activity("nested-only", json!([])),
16191        ])]));
16192        assert!(matches!(
16193            one_leaf.as_mut().poll(&mut task_context),
16194            Poll::Pending
16195        ));
16196        let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16197        assert_eq!(one_leaf_commands.len(), 1);
16198        assert_eq!(
16199            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16200            json!("group")
16201        );
16202        assert_eq!(
16203            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16204            json!(1)
16205        );
16206    }
16207
16208    async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16209        let mut saga = ctx.saga();
16210        let outcome = async {
16211            let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16212            saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16213            let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16214            saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16215            ctx.activity("trip.charge", json!([])).await?;
16216            Ok(json!({"status": "booked"}))
16217        }
16218        .await;
16219        saga.finish(outcome).await
16220    }
16221
16222    fn saga_activity(
16223        event_type: &str,
16224        sequence: u64,
16225        activity_type: &str,
16226        result: Option<Value>,
16227    ) -> HistoryEvent {
16228        let mut payload = json!({
16229            "sequence": sequence,
16230            "activity_type": activity_type,
16231            "message": format!("{activity_type} failed"),
16232            "exception_type": "PlannedFailure",
16233            "non_retryable": true,
16234        });
16235        if let Some(result) = result {
16236            payload["result"] = fixture_envelope(result);
16237        }
16238        history_event(event_type, payload)
16239    }
16240
16241    #[test]
16242    fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16243        let completed_hotel_compensation = saga_activity(
16244            "ActivityCompleted",
16245            4,
16246            "trip.cancel-hotel",
16247            Some(Value::Null),
16248        );
16249        let history = vec![
16250            saga_activity(
16251                "ActivityCompleted",
16252                1,
16253                "trip.reserve-flight",
16254                Some(json!("flight-1")),
16255            ),
16256            saga_activity(
16257                "ActivityCompleted",
16258                2,
16259                "trip.reserve-hotel",
16260                Some(json!("hotel-1")),
16261            ),
16262            saga_activity("ActivityFailed", 3, "trip.charge", None),
16263            completed_hotel_compensation.clone(),
16264            completed_hotel_compensation,
16265        ];
16266
16267        for _restart in 0..2 {
16268            let ctx = workflow_context(history.clone());
16269            let mut future = Box::pin(trip_saga(ctx.clone()));
16270            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16271            assert!(matches!(
16272                future.as_mut().poll(&mut task_context),
16273                Poll::Pending
16274            ));
16275            let commands = ctx.take_commands().expect("compensation command");
16276            assert_eq!(commands.len(), 1);
16277            assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16278        }
16279    }
16280
16281    #[test]
16282    fn saga_compensation_failure_preserves_both_typed_failures() {
16283        let history = vec![
16284            saga_activity(
16285                "ActivityCompleted",
16286                1,
16287                "trip.reserve-flight",
16288                Some(json!("flight-1")),
16289            ),
16290            saga_activity(
16291                "ActivityCompleted",
16292                2,
16293                "trip.reserve-hotel",
16294                Some(json!("hotel-1")),
16295            ),
16296            saga_activity("ActivityFailed", 3, "trip.charge", None),
16297            saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16298        ];
16299        let ctx = workflow_context(history);
16300        let mut future = Box::pin(trip_saga(ctx));
16301        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16302        let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16303            future.as_mut().poll(&mut task_context)
16304        else {
16305            panic!("compensation failure must remain structured");
16306        };
16307        assert!(matches!(
16308            *failure.initiating_failure,
16309            Error::ActivityFailed(_)
16310        ));
16311        assert!(matches!(
16312            *failure.compensation_failure,
16313            Error::ActivityFailed(_)
16314        ));
16315        assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16316        assert_eq!(failure.compensation_registration_order, 2);
16317    }
16318
16319    #[test]
16320    fn saga_compensates_cooperative_cancellation() {
16321        let ctx = workflow_context(vec![saga_activity(
16322            "ActivityCompleted",
16323            1,
16324            "trip.reserve-flight",
16325            Some(json!("flight-1")),
16326        )]);
16327        ctx.state.lock().expect("state").cancel_requested = true;
16328        let run = {
16329            let ctx = ctx.clone();
16330            async move {
16331                let mut saga = ctx.saga();
16332                let outcome = async {
16333                    let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16334                    saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16335                    ctx.throw_if_cancellation_requested()?;
16336                    Ok(json!("unexpected"))
16337                }
16338                .await;
16339                saga.finish(outcome).await
16340            }
16341        };
16342        let mut future = Box::pin(run);
16343        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16344        assert!(matches!(
16345            future.as_mut().poll(&mut task_context),
16346            Poll::Pending
16347        ));
16348        let commands = ctx.take_commands().expect("cancellation compensation");
16349        assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16350    }
16351
16352    fn workflow_task(
16353        workflow_type: &str,
16354        history_events: Vec<HistoryEvent>,
16355        payload_codec: &str,
16356    ) -> WorkflowTask {
16357        WorkflowTask {
16358            task_id: format!("wft-{workflow_type}"),
16359            workflow_command_id: None,
16360            workflow_id: Some(format!("wf-{workflow_type}")),
16361            run_id: Some(format!("run-{workflow_type}")),
16362            workflow_type: workflow_type.to_string(),
16363            cancel_requested: false,
16364            payload_codec: payload_codec.to_string(),
16365            arguments: Some(
16366                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16367            ),
16368            total_history_events: Some(history_events.len() as u64),
16369            history_size_bytes: None,
16370            continue_as_new_recommended: None,
16371            history_budget_pressure: None,
16372            history_events,
16373            next_history_page_token: None,
16374            workflow_task_attempt: 1,
16375            workflow_signal_id: None,
16376            signal_name: None,
16377            signal_arguments: None,
16378            workflow_update_id: None,
16379            update_name: None,
16380            lease_owner: Some("rust-worker".to_string()),
16381        }
16382    }
16383
16384    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16385    struct SideEffectProbe {
16386        request_id: String,
16387        attempt: u32,
16388    }
16389
16390    #[test]
16391    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16392        let calls = AtomicUsize::new(0);
16393        let ctx = workflow_context(Vec::new());
16394        let value = ctx
16395            .side_effect(|| {
16396                calls.fetch_add(1, Ordering::SeqCst);
16397                SideEffectProbe {
16398                    request_id: "request-42".to_string(),
16399                    attempt: 3,
16400                }
16401            })
16402            .expect("first side effect");
16403        assert_eq!(value.attempt, 3);
16404        assert_eq!(calls.load(Ordering::SeqCst), 1);
16405        let commands = ctx.take_commands().expect("commands");
16406        assert_eq!(commands.len(), 1);
16407        assert_eq!(commands[0]["type"], "record_side_effect");
16408        assert_eq!(
16409            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16410            serde_json::to_value(&value).expect("value")
16411        );
16412
16413        let replay = workflow_context(vec![history_event(
16414            "SideEffectRecorded",
16415            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16416        )]);
16417        let replayed: SideEffectProbe = replay
16418            .side_effect(|| {
16419                calls.fetch_add(1, Ordering::SeqCst);
16420                panic!("committed side-effect callbacks must not run during replay")
16421            })
16422            .expect("replayed side effect");
16423        assert_eq!(replayed, value);
16424        assert_eq!(calls.load(Ordering::SeqCst), 1);
16425        assert!(replay.take_commands().expect("commands").is_empty());
16426        replay.ensure_history_consumed().expect("history consumed");
16427    }
16428
16429    #[test]
16430    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16431        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16432        let value = ctx
16433            .side_effect(|| SideEffectProbe {
16434                request_id: "avro-request".to_string(),
16435                attempt: 1,
16436            })
16437            .expect("Avro side effect");
16438        let uuid = ctx.uuid_v4().expect("deterministic UUID");
16439        let commands = ctx.take_commands().expect("commands");
16440        assert_eq!(commands.len(), 2);
16441        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16442        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16443        assert_eq!(
16444            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16445            serde_json::to_value(&value).expect("value")
16446        );
16447
16448        let replay = workflow_context_with_codec(
16449            vec![
16450                history_event(
16451                    "SideEffectRecorded",
16452                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16453                ),
16454                history_event(
16455                    "SideEffectRecorded",
16456                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16457                ),
16458            ],
16459            DEFAULT_CODEC,
16460        );
16461        let replayed: SideEffectProbe = replay
16462            .side_effect(|| panic!("Avro callback must not run"))
16463            .expect("replayed Avro value");
16464        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16465        assert_eq!(replayed, value);
16466        assert_eq!(replayed_uuid, uuid);
16467        assert!(replay.take_commands().expect("commands").is_empty());
16468    }
16469
16470    #[test]
16471    fn typed_side_effect_replay_preserves_bytes_and_maps() {
16472        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16473        let value = ctx
16474            .side_effect_avro_value(typed_fidelity_probe)
16475            .expect("typed side effect");
16476        let commands = ctx.take_commands().expect("side-effect command");
16477        assert_eq!(
16478            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16479                .expect("recorded side effect"),
16480            value
16481        );
16482
16483        let replay = workflow_context_with_codec(
16484            vec![history_event(
16485                "SideEffectRecorded",
16486                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16487            )],
16488            DEFAULT_CODEC,
16489        );
16490        assert_eq!(
16491            replay
16492                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16493                .expect("replayed typed side effect"),
16494            value
16495        );
16496    }
16497
16498    #[test]
16499    fn ordered_side_effects_share_the_durable_command_stream() {
16500        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16501        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16502        let ctx = workflow_context(vec![
16503            history_event(
16504                "SideEffectRecorded",
16505                json!({"sequence": 1, "result": first}),
16506            ),
16507            history_event(
16508                "SideEffectRecorded",
16509                json!({"sequence": 2, "result": second}),
16510            ),
16511        ]);
16512        let first: String = ctx
16513            .side_effect(|| panic!("first callback must not run"))
16514            .expect("first replay");
16515        let second: i32 = ctx
16516            .side_effect(|| panic!("second callback must not run"))
16517            .expect("second replay");
16518        assert_eq!(first, "first");
16519        assert_eq!(second, 29);
16520        ctx.ensure_history_consumed().expect("ordered history");
16521
16522        let reordered = workflow_context(vec![history_event(
16523            "VersionMarkerRecorded",
16524            json!({
16525                "sequence": 1,
16526                "change_id": "before-side-effect",
16527                "version": 1,
16528                "min_supported": 1,
16529                "max_supported": 1,
16530            }),
16531        )]);
16532        let error = reordered
16533            .side_effect(|| "new".to_string())
16534            .expect_err("command reordering must fail");
16535        assert!(matches!(
16536            error,
16537            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16538                if reason == "recorded_command_mismatch"
16539        ));
16540    }
16541
16542    #[test]
16543    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16544        let ctx = workflow_context(Vec::new());
16545        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16546        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16547        assert!(ctx.patched("new-search").expect("patch"));
16548        ctx.deprecate_patch("new-search").expect("deprecate patch");
16549        let commands = ctx.take_commands().expect("commands");
16550        assert_eq!(commands.len(), 2);
16551        assert_eq!(commands[0]["type"], "record_version_marker");
16552        assert_eq!(commands[0]["version"], 2);
16553        assert_eq!(commands[1]["change_id"], "new-search");
16554
16555        let replay = workflow_context(vec![history_event(
16556            "VersionMarkerRecorded",
16557            json!({
16558                "sequence": 1,
16559                "change_id": "checkout-v2",
16560                "version": 2,
16561                "min_supported": 1,
16562                "max_supported": 2,
16563            }),
16564        )]);
16565        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16566        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16567        assert!(replay.take_commands().expect("commands").is_empty());
16568        replay.ensure_history_consumed().expect("history consumed");
16569    }
16570
16571    #[test]
16572    fn version_markers_reject_incompatible_or_malformed_history() {
16573        let incompatible = workflow_context(vec![history_event(
16574            "VersionMarkerRecorded",
16575            json!({
16576                "sequence": 1,
16577                "change_id": "checkout-v2",
16578                "version": 1,
16579                "min_supported": 1,
16580                "max_supported": 2,
16581            }),
16582        )]);
16583        let error = incompatible
16584            .get_version("checkout-v2", 2, 3)
16585            .expect_err("old version is unsupported");
16586        assert!(matches!(
16587            error,
16588            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16589                if reason == "version_marker_incompatible_range"
16590        ));
16591
16592        for (history, reason) in [
16593            (
16594                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16595                "side_effect_result_missing",
16596            ),
16597            (
16598                vec![history_event(
16599                    "SideEffectRecorded",
16600                    json!({
16601                        "sequence": 1,
16602                        "result": {"codec": "avro", "blob": "not-base64"},
16603                    }),
16604                )],
16605                "side_effect_payload_incompatible",
16606            ),
16607            (
16608                vec![history_event(
16609                    "SideEffectRecorded",
16610                    json!({"sequence": 1, "result": {"unwrapped": true}}),
16611                )],
16612                "side_effect_payload_malformed",
16613            ),
16614            (
16615                vec![history_event(
16616                    "VersionMarkerRecorded",
16617                    json!({
16618                        "sequence": 1,
16619                        "change_id": "change",
16620                        "version": 1,
16621                        "min_supported": 2,
16622                        "max_supported": 1,
16623                    }),
16624                )],
16625                "version_marker_history_range_invalid",
16626            ),
16627        ] {
16628            let error = WorkflowState::new(
16629                history,
16630                "rust-workers".to_string(),
16631                DEFAULT_CODEC.to_string(),
16632                None,
16633            )
16634            .expect_err("malformed history must fail");
16635            assert!(matches!(
16636                error,
16637                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16638                    if actual == reason
16639            ));
16640        }
16641    }
16642
16643    #[test]
16644    fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16645        let history = vec![history_event(
16646            "SearchAttributesUpserted",
16647            json!({
16648                "sequence": 1,
16649                "attributes": {"customer_tier": "gold"},
16650                "attribute_types": {"customer_tier": "keyword"},
16651                "merged": {"customer_tier": "gold"}
16652            }),
16653        )];
16654
16655        let matching = workflow_context(history.clone());
16656        matching
16657            .upsert_search_attributes(
16658                SearchAttributeUpdate::new()
16659                    .keyword("customer_tier", "gold")
16660                    .expect("keyword update"),
16661            )
16662            .expect("matching typed update must replay");
16663        matching
16664            .ensure_history_consumed()
16665            .expect("history consumed");
16666
16667        let changed_type = workflow_context(history.clone());
16668        let error = changed_type
16669            .upsert_search_attributes(
16670                SearchAttributeUpdate::new()
16671                    .string("customer_tier", "gold")
16672                    .expect("string update"),
16673            )
16674            .expect_err("same JSON value with a different declaration must be nondeterministic");
16675        let Error::NonDeterministicReplay(failure) = error else {
16676            panic!("typed identity drift must be a replay failure");
16677        };
16678        assert_eq!(failure.reason, "search_attribute_type_mismatch");
16679        assert_eq!(failure.sequence, Some(1));
16680
16681        let changed_value = workflow_context(history);
16682        let error = changed_value
16683            .upsert_search_attributes(
16684                SearchAttributeUpdate::new()
16685                    .keyword("customer_tier", "platinum")
16686                    .expect("keyword update"),
16687            )
16688            .expect_err("changed values must be nondeterministic");
16689        let Error::NonDeterministicReplay(failure) = error else {
16690            panic!("value drift must be a replay failure");
16691        };
16692        assert_eq!(failure.reason, "search_attribute_value_mismatch");
16693    }
16694
16695    #[test]
16696    fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16697        let history = vec![history_event(
16698            "SearchAttributesUpserted",
16699            json!({
16700                "sequence": 1,
16701                "attributes": {"customer_tier": "gold"},
16702                "merged": {"customer_tier": "gold"}
16703            }),
16704        )];
16705
16706        for update in [
16707            SearchAttributeUpdate::new()
16708                .keyword("customer_tier", "gold")
16709                .expect("keyword update"),
16710            SearchAttributeUpdate::new()
16711                .string("customer_tier", "gold")
16712                .expect("string update"),
16713        ] {
16714            let restarted = workflow_context(history.clone());
16715            restarted
16716                .upsert_search_attributes(update)
16717                .expect("legacy history constrains values but has unknown type identity");
16718            restarted
16719                .ensure_history_consumed()
16720                .expect("history consumed");
16721        }
16722    }
16723
16724    #[test]
16725    fn search_attribute_command_emits_canonical_types() {
16726        let ctx = workflow_context(Vec::new());
16727        ctx.upsert_search_attributes(
16728            SearchAttributeUpdate::new()
16729                .keyword("customer_tier", "gold")
16730                .expect("keyword update")
16731                .int("attempts", 3)
16732                .expect("int update")
16733                .delete("obsolete")
16734                .expect("delete update"),
16735        )
16736        .expect("valid search attributes");
16737
16738        assert_eq!(
16739            ctx.take_commands().expect("commands"),
16740            vec![json!({
16741                "type": "upsert_search_attributes",
16742                "attributes": {
16743                    "attempts": 3,
16744                    "customer_tier": "gold",
16745                    "obsolete": null
16746                },
16747                "attribute_types": {
16748                    "attempts": "int",
16749                    "customer_tier": "keyword"
16750                }
16751            })]
16752        );
16753    }
16754
16755    #[test]
16756    fn duplicate_side_effects_and_version_markers_are_rejected() {
16757        let duplicate_side_effect = WorkflowState::new(
16758            vec![
16759                history_event(
16760                    "SideEffectRecorded",
16761                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16762                ),
16763                history_event(
16764                    "SideEffectRecorded",
16765                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16766                ),
16767            ],
16768            "rust-workers".to_string(),
16769            DEFAULT_CODEC.to_string(),
16770            None,
16771        )
16772        .expect_err("duplicate side effect");
16773        assert!(matches!(
16774            duplicate_side_effect,
16775            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16776                if reason == "duplicate_side_effect_record"
16777        ));
16778
16779        let marker = |sequence| {
16780            history_event(
16781                "VersionMarkerRecorded",
16782                json!({
16783                    "sequence": sequence,
16784                    "change_id": "same-change",
16785                    "version": 1,
16786                    "min_supported": 1,
16787                    "max_supported": 1,
16788                }),
16789            )
16790        };
16791        let duplicate_marker = WorkflowState::new(
16792            vec![marker(1), marker(3)],
16793            "rust-workers".to_string(),
16794            DEFAULT_CODEC.to_string(),
16795            None,
16796        )
16797        .expect_err("duplicate marker");
16798        assert!(matches!(
16799            duplicate_marker,
16800            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16801                if reason == "duplicate_version_marker"
16802        ));
16803    }
16804
16805    #[test]
16806    fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16807        let mut state = WorkflowState::new(
16808            Vec::new(),
16809            "rust-workers".to_string(),
16810            DEFAULT_CODEC.to_string(),
16811            None,
16812        )
16813        .expect("workflow state");
16814        state.workflow_command_identity = "command-7".to_string();
16815        let context = WorkflowContext {
16816            state: Arc::new(Mutex::new(state)),
16817        };
16818        let item =
16819            WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16820
16821        context
16822            .append_workflow_stream("output", &[item], Some(10))
16823            .expect("append command");
16824        context
16825            .error_workflow_stream("output", "producer failed", None)
16826            .expect("error command");
16827        let commands = context.take_commands().expect("commands");
16828
16829        assert_eq!(commands[0]["type"], "record_side_effect");
16830        assert_eq!(
16831            commands[0]["workflow_stream"]["command_identity"],
16832            "command-7"
16833        );
16834        assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16835        assert_eq!(
16836            commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16837            "dw-stream:command-7:0:0"
16838        );
16839        assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16840
16841        let recorded = history_event(
16842            "SideEffectRecorded",
16843            json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16844        );
16845        let mut replay_state = WorkflowState::new(
16846            vec![recorded],
16847            "rust-workers".to_string(),
16848            DEFAULT_CODEC.to_string(),
16849            None,
16850        )
16851        .expect("replay state");
16852        replay_state.workflow_command_identity = "command-7".to_string();
16853        let replay_context = WorkflowContext {
16854            state: Arc::new(Mutex::new(replay_state)),
16855        };
16856        replay_context
16857            .append_workflow_stream(
16858                "output",
16859                &[WorkflowStreamAppendItem::from_reference(
16860                    "s3://bucket/item.avro",
16861                )],
16862                Some(10),
16863            )
16864            .expect("replayed append");
16865        assert!(replay_context
16866            .take_commands()
16867            .expect("replayed commands")
16868            .is_empty());
16869    }
16870
16871    #[test]
16872    fn workflow_stream_authoring_requires_server_durable_command_identity() {
16873        let context = workflow_context(Vec::new());
16874        let error = context
16875            .append_workflow_stream(
16876                "output",
16877                &[WorkflowStreamAppendItem::from_reference(
16878                    "s3://bucket/item.avro",
16879                )],
16880                None,
16881            )
16882            .expect_err("stream append without durable command identity must fail closed");
16883
16884        assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16885        assert!(context.take_commands().expect("commands").is_empty());
16886    }
16887
16888    #[test]
16889    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16890        fn worker(calls: Arc<AtomicUsize>) -> Worker {
16891            let client = Client::new("http://127.0.0.1:8080").expect("client");
16892            let mut worker = Worker::new(client, "rust-workers");
16893            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16894                let calls = Arc::clone(&calls);
16895                async move {
16896                    let captured = ctx.side_effect(|| {
16897                        calls.fetch_add(1, Ordering::SeqCst);
16898                        "captured-once".to_string()
16899                    })?;
16900                    let version = ctx.get_version("cold-restart", 1, 2)?;
16901                    Ok(json!({"captured": captured, "version": version}))
16902                }
16903            });
16904            worker
16905        }
16906
16907        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16908            WorkflowTask {
16909                task_id: "wft-side-effect-version".to_string(),
16910                workflow_command_id: None,
16911                workflow_id: Some("wf-side-effect-version".to_string()),
16912                run_id: Some("run-side-effect-version".to_string()),
16913                workflow_type: "rust.side-effect-version".to_string(),
16914                cancel_requested: false,
16915                payload_codec: DEFAULT_CODEC.to_string(),
16916                arguments: Some(
16917                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16918                ),
16919                history_events,
16920                total_history_events: None,
16921                history_size_bytes: None,
16922                continue_as_new_recommended: None,
16923                history_budget_pressure: None,
16924                next_history_page_token: None,
16925                workflow_task_attempt: 1,
16926                workflow_signal_id: None,
16927                signal_name: None,
16928                signal_arguments: None,
16929                workflow_update_id: None,
16930                update_name: None,
16931                lease_owner: Some("rust-worker".to_string()),
16932            }
16933        }
16934
16935        let calls = Arc::new(AtomicUsize::new(0));
16936        let initial = worker(Arc::clone(&calls))
16937            .execute_workflow_task(task(Vec::new()))
16938            .expect("initial execution");
16939        assert_eq!(
16940            initial
16941                .iter()
16942                .map(|command| &command["type"])
16943                .collect::<Vec<_>>(),
16944            vec![
16945                "record_side_effect",
16946                "record_version_marker",
16947                "complete_workflow"
16948            ]
16949        );
16950        assert_eq!(calls.load(Ordering::SeqCst), 1);
16951
16952        let restarted = worker(Arc::clone(&calls));
16953        let replayed = restarted
16954            .execute_workflow_task(task(vec![
16955                history_event(
16956                    "SideEffectRecorded",
16957                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16958                ),
16959                history_event(
16960                    "VersionMarkerRecorded",
16961                    json!({
16962                        "sequence": 2,
16963                        "change_id": "cold-restart",
16964                        "version": 2,
16965                        "min_supported": 1,
16966                        "max_supported": 2,
16967                    }),
16968                ),
16969            ]))
16970            .expect("cold replay");
16971        assert_eq!(replayed.len(), 1);
16972        assert_eq!(replayed[0]["type"], "complete_workflow");
16973        assert_eq!(calls.load(Ordering::SeqCst), 1);
16974    }
16975
16976    #[test]
16977    fn side_effect_replay_rejects_changed_rust_value_type() {
16978        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16979        let ctx = workflow_context(vec![history_event(
16980            "SideEffectRecorded",
16981            json!({"sequence": 1, "result": result}),
16982        )]);
16983        let error = ctx
16984            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16985            .expect_err("changed type must fail replay");
16986        assert!(matches!(
16987            error,
16988            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16989                if reason == "side_effect_type_mismatch"
16990        ));
16991    }
16992
16993    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16994        vec![
16995            history_event(
16996                "ActivityScheduled",
16997                json!({
16998                    "sequence": 1,
16999                    "activity_type": "flaky",
17000                    "activity_execution_id": "act-1",
17001                    "activity": {
17002                        "id": "act-1",
17003                        "sequence": 1,
17004                        "type": "flaky",
17005                        "queue": "critical-activities",
17006                        "execution_mode": null,
17007                        "retry_policy": {
17008                            "snapshot_version": 1,
17009                            "max_attempts": 3,
17010                            "backoff_seconds": [2, 4],
17011                            "start_to_close_timeout": 30,
17012                            "schedule_to_start_timeout": 5,
17013                            "schedule_to_close_timeout": 90,
17014                            "heartbeat_timeout": 10,
17015                            "non_retryable_error_types": ["PermanentError"]
17016                        }
17017                    }
17018                }),
17019            ),
17020            history_event(
17021                "ActivityStarted",
17022                json!({
17023                    "sequence": 1,
17024                    "activity_type": "flaky",
17025                    "activity_execution_id": "act-1",
17026                    "activity_attempt_id": "attempt-1",
17027                    "attempt_number": 1
17028                }),
17029            ),
17030            history_event(
17031                "ActivityRetryScheduled",
17032                json!({
17033                    "sequence": 1,
17034                    "activity_type": "flaky",
17035                    "activity_execution_id": "act-1",
17036                    "activity_attempt_id": "attempt-1",
17037                    "attempt_number": 1,
17038                    "retry_after_attempt": 1,
17039                    "retry_backoff_seconds": 2,
17040                    "failure_category": "activity",
17041                    "exception_type": "TransientError"
17042                }),
17043            ),
17044            history_event(
17045                "ActivityStarted",
17046                json!({
17047                    "sequence": 1,
17048                    "activity_type": "flaky",
17049                    "activity_execution_id": "act-1",
17050                    "activity_attempt_id": "attempt-2",
17051                    "attempt_number": 2
17052                }),
17053            ),
17054            history_event(
17055                "ActivityCompleted",
17056                json!({
17057                    "sequence": 1,
17058                    "activity_type": "flaky",
17059                    "activity_execution_id": "act-1",
17060                    "activity_attempt_id": "attempt-2",
17061                    "attempt_number": 2,
17062                    "payload_codec": DEFAULT_CODEC,
17063                    "result": fixture_envelope(json!({"status":"recovered"}))
17064                }),
17065            ),
17066        ]
17067    }
17068
17069    fn retry_activity_options() -> ActivityOptions {
17070        ActivityOptions::new()
17071            .task_queue("critical-activities")
17072            .retry_policy(
17073                ActivityRetryPolicy::new(3)
17074                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
17075                    .non_retryable_error_type("PermanentError"),
17076            )
17077            .start_to_close_timeout(Duration::from_secs(30))
17078            .schedule_to_start_timeout(Duration::from_secs(5))
17079            .schedule_to_close_timeout(Duration::from_secs(90))
17080            .heartbeat_timeout(Duration::from_secs(10))
17081    }
17082
17083    #[test]
17084    fn fixed_avro_value_round_trips_json_values() {
17085        let value = json!({"greeting": "hello", "count": 3, "ok": true});
17086        let envelope = PayloadEnvelope::avro(&value).expect("encode");
17087        assert_eq!(envelope.codec, DEFAULT_CODEC);
17088        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
17089    }
17090
17091    #[tokio::test]
17092    async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
17093        let client = Client::new("http://127.0.0.1:8080").expect("client");
17094        let mut worker = Worker::new(client, "rust-workers");
17095        worker.register_typed_workflow(
17096            "typed.contract.workflow",
17097            |_ctx, input: TypedContract| async move { Ok(input) },
17098        );
17099        worker.register_typed_activity(
17100            "typed.contract.activity",
17101            |_ctx, input: TypedContract| async move { Ok(input) },
17102        );
17103
17104        let expected = typed_contract();
17105        let arguments = AvroValue::Array(vec![
17106            AvroValue::from_serialize(&expected).expect("typed request")
17107        ]);
17108        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
17109        let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
17110        workflow.arguments = Some(envelope.clone());
17111        let commands = worker
17112            .execute_workflow_task(workflow)
17113            .expect("typed workflow task");
17114        let workflow_result: TypedContract =
17115            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17116                .expect("workflow result envelope")
17117                .deserialize()
17118                .expect("workflow result type");
17119        assert_eq!(workflow_result, expected);
17120
17121        let activity = ActivityTask {
17122            task_id: "typed-contract-activity".to_string(),
17123            activity_attempt_id: Some("typed-contract-attempt".to_string()),
17124            attempt_id: None,
17125            activity_type: "typed.contract.activity".to_string(),
17126            payload_codec: DEFAULT_CODEC.to_string(),
17127            arguments: Some(envelope),
17128            attempt_number: 1,
17129            lease_owner: Some("rust-worker".to_string()),
17130        };
17131        let activity_result: TypedContract = worker
17132            .execute_activity_task(activity)
17133            .await
17134            .expect("typed activity task")
17135            .deserialize()
17136            .expect("activity result type");
17137        assert_eq!(activity_result, expected);
17138    }
17139
17140    #[tokio::test]
17141    async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17142        let client = Client::new("http://127.0.0.1:8080").expect("client");
17143        let mut worker = Worker::new(client, "rust-workers");
17144        worker.register_typed_workflow(
17145            "typed.shape.workflow",
17146            |_ctx, input: TypedContract| async move { Ok(input) },
17147        );
17148        worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17149            Ok(f64::NAN)
17150        });
17151
17152        let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17153        workflow.arguments = Some(
17154            encode_typed_envelope(
17155                &AvroValue::Array(vec![
17156                    AvroValue::String("first".to_string()),
17157                    AvroValue::String("second".to_string()),
17158                ]),
17159                DEFAULT_CODEC,
17160            )
17161            .expect("malformed typed arguments"),
17162        );
17163        let commands = worker
17164            .execute_workflow_task(workflow)
17165            .expect("shape mismatch becomes a workflow failure");
17166        let message = commands[0]["message"].as_str().expect("failure message");
17167        assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17168        assert!(message.contains(type_name::<TypedContract>()));
17169        assert!(message.contains("task carried 2 arguments"));
17170
17171        let activity = ActivityTask {
17172            task_id: "typed-unsupported-activity".to_string(),
17173            activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17174            attempt_id: None,
17175            activity_type: "typed.unsupported.activity".to_string(),
17176            payload_codec: DEFAULT_CODEC.to_string(),
17177            arguments: Some(
17178                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17179                    .expect("unit arguments"),
17180            ),
17181            attempt_number: 1,
17182            lease_owner: Some("rust-worker".to_string()),
17183        };
17184        let Error::HandlerType {
17185            handler_kind,
17186            handler_name,
17187            value_kind,
17188            rust_type,
17189            message,
17190        } = worker
17191            .execute_activity_task(activity)
17192            .await
17193            .expect_err("non-finite handler output must fail")
17194        else {
17195            panic!("expected contextual handler type failure");
17196        };
17197        assert_eq!(handler_kind, HandlerKind::Activity);
17198        assert_eq!(handler_name, "typed.unsupported.activity");
17199        assert_eq!(value_kind, HandlerValueKind::Result);
17200        assert_eq!(rust_type, type_name::<f64>());
17201        assert!(message.contains("non_finite_float"));
17202    }
17203
17204    #[tokio::test]
17205    async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17206        #[derive(Clone, Default)]
17207        struct State {
17208            observed: Option<TypedContract>,
17209        }
17210
17211        let client = Client::new("http://127.0.0.1:8080").expect("client");
17212        let mut worker = Worker::new(client, "rust-workers");
17213        worker.register_typed_replayed_workflow(
17214            "typed.contract.replayed",
17215            State::default,
17216            |ctx, input: TypedContract, state| async move {
17217                let result: TypedContract =
17218                    ctx.activity_typed("typed.contract.activity", input).await?;
17219                state.update(|current| current.observed = Some(result.clone()))?;
17220                Ok(result)
17221            },
17222        );
17223        worker.register_replayed_query::<State, _, _>(
17224            "typed.contract.replayed",
17225            "observed",
17226            |_ctx, state, _args| async move {
17227                Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17228            },
17229        );
17230
17231        let expected = typed_contract();
17232        let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17233        let workflow_arguments =
17234            encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17235                .expect("workflow arguments");
17236        let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17237        let task = QueryTask {
17238            query_task_id: "typed-replay-query".to_string(),
17239            query_task_attempt: 1,
17240            lease_owner: Some("rust-worker".to_string()),
17241            workflow_id: Some("typed-replay".to_string()),
17242            run_id: Some("typed-replay-run".to_string()),
17243            workflow_type: "typed.contract.replayed".to_string(),
17244            query_name: "observed".to_string(),
17245            payload_codec: DEFAULT_CODEC.to_string(),
17246            workflow_arguments: Some(workflow_arguments),
17247            query_arguments: Some(
17248                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17249                    .expect("query arguments"),
17250            ),
17251            history_events: vec![
17252                history_event(
17253                    "ActivityScheduled",
17254                    json!({
17255                        "sequence": 1,
17256                        "activity_type": "typed.contract.activity"
17257                    }),
17258                ),
17259                history_event(
17260                    "ActivityCompleted",
17261                    json!({
17262                        "sequence": 1,
17263                        "activity_type": "typed.contract.activity",
17264                        "payload_codec": DEFAULT_CODEC,
17265                        "result": result
17266                    }),
17267                ),
17268            ],
17269            history_export: None,
17270            run_status: Some("completed".to_string()),
17271        };
17272
17273        assert_eq!(
17274            worker
17275                .execute_query_task(task)
17276                .await
17277                .expect("typed replay query")
17278                .deserialize::<i64>()
17279                .expect("query result"),
17280            expected.signed
17281        );
17282    }
17283
17284    #[tokio::test]
17285    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17286        let client = Client::new("http://127.0.0.1:8080").expect("client");
17287        let mut worker = Worker::new(client, "rust-workers");
17288        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17289        worker
17290            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17291        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17292            Ok(input)
17293        });
17294        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17295            Ok(input)
17296        });
17297        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17298            Ok(AvroValue::Array(
17299                ctx.wait_signal_avro_value("changed").await?,
17300            ))
17301        });
17302
17303        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17304        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17305
17306        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17307        workflow.arguments = Some(envelope.clone());
17308        let commands = worker
17309            .execute_workflow_task(workflow)
17310            .expect("typed workflow task");
17311        assert_eq!(commands[0]["type"], "complete_workflow");
17312        assert_eq!(
17313            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17314                .expect("typed workflow result"),
17315            arguments
17316        );
17317
17318        let activity = ActivityTask {
17319            task_id: "activity-typed".to_string(),
17320            activity_attempt_id: Some("attempt-typed".to_string()),
17321            attempt_id: None,
17322            activity_type: "typed.activity".to_string(),
17323            payload_codec: DEFAULT_CODEC.to_string(),
17324            arguments: Some(envelope.clone()),
17325            attempt_number: 1,
17326            lease_owner: Some("rust-worker".to_string()),
17327        };
17328        assert_eq!(
17329            worker
17330                .execute_activity_task(activity)
17331                .await
17332                .expect("typed activity result"),
17333            arguments
17334        );
17335
17336        let query = QueryTask {
17337            query_task_id: "query-typed".to_string(),
17338            query_task_attempt: 1,
17339            lease_owner: Some("rust-worker".to_string()),
17340            workflow_id: Some("typed-1".to_string()),
17341            run_id: Some("run-typed".to_string()),
17342            workflow_type: "typed.echo".to_string(),
17343            query_name: "inspect".to_string(),
17344            payload_codec: DEFAULT_CODEC.to_string(),
17345            workflow_arguments: Some(
17346                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17347                    .expect("workflow input"),
17348            ),
17349            query_arguments: Some(envelope.clone()),
17350            history_events: Vec::new(),
17351            history_export: None,
17352            run_status: Some("running".to_string()),
17353        };
17354        assert_eq!(
17355            worker
17356                .execute_query_task(query)
17357                .await
17358                .expect("typed query result"),
17359            arguments
17360        );
17361
17362        let mut update = workflow_task(
17363            "typed.echo",
17364            vec![history_event(
17365                "UpdateAccepted",
17366                json!({
17367                    "update_id": "update-typed",
17368                    "update_name": "replace",
17369                    "arguments": envelope.clone(),
17370                }),
17371            )],
17372            DEFAULT_CODEC,
17373        );
17374        update.workflow_update_id = Some("update-typed".to_string());
17375        update.update_name = Some("replace".to_string());
17376        let commands = worker
17377            .execute_workflow_task(update)
17378            .expect("typed update task");
17379        assert_eq!(commands[0]["type"], "complete_update");
17380        assert_eq!(
17381            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17382                .expect("typed update result"),
17383            arguments
17384        );
17385
17386        let mut signal = workflow_task(
17387            "typed.signal",
17388            vec![history_event(
17389                "SignalReceived",
17390                json!({
17391                    "signal_id": "signal-typed",
17392                    "signal_name": "changed",
17393                    "arguments": envelope.clone(),
17394                }),
17395            )],
17396            DEFAULT_CODEC,
17397        );
17398        signal.workflow_signal_id = Some("signal-typed".to_string());
17399        signal.signal_name = Some("changed".to_string());
17400        signal.signal_arguments = Some(envelope);
17401        let commands = worker
17402            .execute_workflow_task(signal)
17403            .expect("typed signal resume");
17404        assert_eq!(
17405            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17406                .expect("typed signal result"),
17407            arguments
17408        );
17409    }
17410
17411    #[tokio::test]
17412    async fn typed_helpers_never_parse_json_inspection_projection() {
17413        let collision_values = projection_collision_probe();
17414        let expected = AvroValue::Array(collision_values.clone());
17415        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17416
17417        let activity_context = workflow_context_with_codec(
17418            vec![history_event(
17419                "ActivityCompleted",
17420                json!({
17421                    "sequence": 1,
17422                    "activity_type": "collision.activity",
17423                    "payload_codec": DEFAULT_CODEC,
17424                    "result": envelope.clone(),
17425                }),
17426            )],
17427            DEFAULT_CODEC,
17428        );
17429        assert_eq!(
17430            activity_context
17431                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17432                .await
17433                .expect("typed activity collision result"),
17434            expected
17435        );
17436
17437        let signal_context = workflow_context_with_codec(
17438            vec![
17439                history_event(
17440                    "SignalWaitOpened",
17441                    json!({"sequence": 1, "signal_name": "collision"}),
17442                ),
17443                history_event(
17444                    "SignalApplied",
17445                    json!({
17446                        "sequence": 1,
17447                        "signal_name": "collision",
17448                        "payload_codec": DEFAULT_CODEC,
17449                        "value": envelope.clone(),
17450                    }),
17451                ),
17452            ],
17453            DEFAULT_CODEC,
17454        );
17455        assert_eq!(
17456            signal_context
17457                .wait_signal_avro_value("collision")
17458                .await
17459                .expect("typed signal collision arguments"),
17460            collision_values
17461        );
17462
17463        let child_context = workflow_context_with_codec(
17464            vec![
17465                history_event(
17466                    "ChildWorkflowScheduled",
17467                    json!({
17468                        "sequence": 1,
17469                        "child_workflow_instance_id": "collision-child",
17470                        "child_workflow_run_id": "collision-run",
17471                        "child_workflow_type": "collision.child",
17472                    }),
17473                ),
17474                history_event(
17475                    "ChildRunCompleted",
17476                    json!({
17477                        "sequence": 1,
17478                        "child_workflow_instance_id": "collision-child",
17479                        "child_workflow_run_id": "collision-run",
17480                        "child_workflow_type": "collision.child",
17481                        "payload_codec": DEFAULT_CODEC,
17482                        "result": envelope,
17483                    }),
17484                ),
17485            ],
17486            DEFAULT_CODEC,
17487        );
17488        let child = child_context
17489            .start_child_workflow_avro_value(
17490                "collision.child",
17491                ChildWorkflowOptions::new("collision-workers"),
17492                AvroValue::Array(Vec::new()),
17493            )
17494            .await
17495            .expect("typed child collision result");
17496        assert_eq!(child.result, expected);
17497    }
17498
17499    #[tokio::test]
17500    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17501        let client = Client::new("http://127.0.0.1:8080").expect("client");
17502        let mut worker = Worker::new(client, "rust-workers");
17503        worker.register_replayed_workflow_avro_value(
17504            "typed.replayed",
17505            || (),
17506            |_ctx, input, _state| async move { Ok(input) },
17507        );
17508        worker.register_replayed_query_avro_value::<(), _, _>(
17509            "typed.replayed",
17510            "inspect",
17511            |ctx, _state, args| async move {
17512                let mut signals = ctx.signals_avro_value("collision");
17513                let signal = signals
17514                    .pop()
17515                    .map(AvroValue::Array)
17516                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17517                Ok(AvroValue::Array(vec![
17518                    ctx.workflow_input_avro_value().clone(),
17519                    signal,
17520                    args,
17521                ]))
17522            },
17523        );
17524        let arguments = AvroValue::Array(projection_collision_probe());
17525        let signal_arguments =
17526            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17527        let task = QueryTask {
17528            query_task_id: "query-typed-replay".to_string(),
17529            query_task_attempt: 1,
17530            lease_owner: Some("rust-worker".to_string()),
17531            workflow_id: Some("typed-replay".to_string()),
17532            run_id: Some("run-typed-replay".to_string()),
17533            workflow_type: "typed.replayed".to_string(),
17534            query_name: "inspect".to_string(),
17535            payload_codec: DEFAULT_CODEC.to_string(),
17536            workflow_arguments: Some(
17537                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17538            ),
17539            query_arguments: Some(
17540                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17541            ),
17542            history_events: vec![history_event(
17543                "SignalReceived",
17544                json!({
17545                    "signal_id": "collision-signal",
17546                    "signal_name": "collision",
17547                    "workflow_sequence": 1,
17548                    "payload_codec": DEFAULT_CODEC,
17549                    "arguments": signal_arguments,
17550                }),
17551            )],
17552            history_export: None,
17553            run_status: Some("completed".to_string()),
17554        };
17555
17556        assert_eq!(
17557            worker
17558                .execute_query_task(task)
17559                .await
17560                .expect("typed replay query"),
17561            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17562        );
17563    }
17564
17565    #[test]
17566    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17567        let value = BTreeMap::from([(1_i32, "integer key")]);
17568        let error = PayloadEnvelope::avro(&value)
17569            .expect_err("integer map keys must fail")
17570            .to_string();
17571
17572        assert!(error.contains("invalid_map_key"));
17573    }
17574
17575    #[test]
17576    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17577        let envelope = PayloadEnvelope {
17578            codec: "json".to_string(),
17579            blob: r#"{"greeting":"hello"}"#.to_string(),
17580        };
17581
17582        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17583        let diagnostic = error.to_string();
17584        assert!(diagnostic.contains("unsupported_payload_codec"));
17585        assert!(diagnostic.contains("codec=\"avro\""));
17586        assert!(diagnostic.contains("HTTP document transport"));
17587    }
17588
17589    #[test]
17590    fn untagged_json_payload_value_fails_closed() {
17591        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17592            .expect_err("untagged JSON payload values must fail");
17593        let diagnostic = error.to_string();
17594        assert!(diagnostic.contains("unsupported_payload_codec"));
17595        assert!(diagnostic.contains("untagged durable payload"));
17596        assert!(diagnostic.contains("HTTP document transport"));
17597    }
17598
17599    #[test]
17600    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17601        let envelope = PayloadEnvelope {
17602            codec: DEFAULT_CODEC.to_string(),
17603            blob: BASE64.encode([0x01]),
17604        };
17605
17606        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17607        assert!(error.to_string().contains("invalid_payload_framing"));
17608    }
17609
17610    #[tokio::test]
17611    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17612        let server = MockWorkerServer::start();
17613        let client = Client::builder(server.base_url())
17614            .timeout(Duration::from_secs(2))
17615            .build()
17616            .expect("client");
17617        let invalid_commands = [
17618            json!({
17619                "type": "complete_workflow",
17620                "result": {"codec": "json", "blob": null}
17621            }),
17622            json!({
17623                "type": "schedule_activity",
17624                "arguments": {"codec": "yaml", "blob": "ignored"}
17625            }),
17626            json!({
17627                "type": "start_child_workflow",
17628                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17629            }),
17630            json!({"type": "continue_as_new", "arguments": []}),
17631            json!({"type": "complete_update"}),
17632            json!({"type": "record_side_effect", "result": null}),
17633            json!({
17634                "type": "start_service_operation",
17635                "payload_codec": DEFAULT_CODEC,
17636                "request_payload": "raw-avro-bytes"
17637            }),
17638        ];
17639
17640        for command in invalid_commands {
17641            let error = client
17642                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17643                .await
17644                .expect_err("invalid durable payload must fail locally");
17645            let diagnostic = error.to_string();
17646            assert!(
17647                diagnostic.contains("unsupported_payload_codec")
17648                    || diagnostic.contains("invalid_payload_envelope")
17649                    || diagnostic.contains("untagged durable payload"),
17650                "unexpected validation diagnostic: {diagnostic}"
17651            );
17652        }
17653
17654        assert_eq!(
17655            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17656            0,
17657            "invalid command payloads must not reach HTTP transport"
17658        );
17659    }
17660
17661    #[test]
17662    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17663        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17664        let commands = [
17665            json!({"type": "complete_workflow", "result": envelope.clone()}),
17666            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17667            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17668            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17669            json!({"type": "complete_update", "result": envelope.clone()}),
17670            json!({"type": "record_side_effect", "result": envelope.clone()}),
17671            json!({
17672                "type": "start_service_operation",
17673                "payload_codec": DEFAULT_CODEC,
17674                "request_payload": envelope.clone()
17675            }),
17676            json!({
17677                "type": "complete_workflow",
17678                "result": envelope,
17679                "metadata": {
17680                    "codec": "json",
17681                    "payload_codec": "customer-codec",
17682                    "result": {"codec": "yaml", "blob": null}
17683                }
17684            }),
17685        ];
17686
17687        validate_workflow_task_commands(&commands)
17688            .expect("customer metadata must not become a protocol codec declaration");
17689    }
17690
17691    #[test]
17692    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17693        assert_eq!(
17694            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17695            AvroValue::Array(Vec::new())
17696        );
17697        assert_eq!(
17698            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17699            AvroValue::Array(Vec::new())
17700        );
17701
17702        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17703        signal.signal_name = Some("empty-signal".to_string());
17704        signal.signal_arguments = None;
17705        let decoded = decode_resume_signal(&signal)
17706            .expect("valid Avro signal")
17707            .expect("named signal resumes the workflow");
17708        assert!(decoded.arguments.is_empty());
17709    }
17710
17711    #[tokio::test]
17712    async fn malformed_task_level_codecs_become_pre_handler_failures() {
17713        let client = Client::new("http://127.0.0.1:8080").expect("client");
17714        let mut worker = Worker::new(client, "rust-workers");
17715        let handler_calls = Arc::new(AtomicUsize::new(0));
17716
17717        let calls = Arc::clone(&handler_calls);
17718        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17719            calls.fetch_add(1, Ordering::SeqCst);
17720            async move { Ok(Value::Null) }
17721        });
17722        let calls = Arc::clone(&handler_calls);
17723        worker.register_activity("codec.activity", move |_ctx, _args| {
17724            calls.fetch_add(1, Ordering::SeqCst);
17725            async move { Ok(Value::Null) }
17726        });
17727        let calls = Arc::clone(&handler_calls);
17728        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17729            calls.fetch_add(1, Ordering::SeqCst);
17730            async move { Ok(Value::Null) }
17731        });
17732
17733        let mut failures = Vec::new();
17734        for codec_case in [
17735            InvalidTaskPayloadCodec::Missing,
17736            InvalidTaskPayloadCodec::Null,
17737            InvalidTaskPayloadCodec::NonString,
17738        ] {
17739            let mut workflow = json!({
17740                "task_id": format!("workflow-{}", codec_case.label()),
17741                "workflow_type": "codec.workflow"
17742            });
17743            codec_case.apply(&mut workflow);
17744            match serde_json::from_value::<WorkflowTask>(workflow) {
17745                Ok(task) => match worker.execute_workflow_task(task) {
17746                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17747                    outcome => failures.push(format!(
17748                        "workflow {} codec returned {outcome:?}",
17749                        codec_case.label()
17750                    )),
17751                },
17752                Err(error) => failures.push(format!(
17753                    "workflow {} codec failed transport deserialization: {error}",
17754                    codec_case.label()
17755                )),
17756            }
17757
17758            let mut activity = json!({
17759                "task_id": format!("activity-{}", codec_case.label()),
17760                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17761                "activity_type": "codec.activity",
17762                "attempt_number": 1
17763            });
17764            codec_case.apply(&mut activity);
17765            match serde_json::from_value::<ActivityTask>(activity) {
17766                Ok(task) => match worker.execute_activity_task(task).await {
17767                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17768                    outcome => failures.push(format!(
17769                        "activity {} codec returned {outcome:?}",
17770                        codec_case.label()
17771                    )),
17772                },
17773                Err(error) => failures.push(format!(
17774                    "activity {} codec failed transport deserialization: {error}",
17775                    codec_case.label()
17776                )),
17777            }
17778
17779            let mut query = json!({
17780                "query_task_id": format!("query-{}", codec_case.label()),
17781                "workflow_type": "codec.workflow",
17782                "query_name": "known"
17783            });
17784            codec_case.apply(&mut query);
17785            match serde_json::from_value::<QueryTask>(query) {
17786                Ok(task) => match worker.execute_query_task(task).await {
17787                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17788                    outcome => failures.push(format!(
17789                        "query {} codec returned {outcome:?}",
17790                        codec_case.label()
17791                    )),
17792                },
17793                Err(error) => failures.push(format!(
17794                    "query {} codec failed transport deserialization: {error}",
17795                    codec_case.label()
17796                )),
17797            }
17798        }
17799
17800        assert!(failures.is_empty(), "{}", failures.join("\n"));
17801        assert_eq!(
17802            handler_calls.load(Ordering::SeqCst),
17803            0,
17804            "invalid task codecs must not invoke a handler"
17805        );
17806    }
17807
17808    #[tokio::test]
17809    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17810        for codec_case in [
17811            InvalidTaskPayloadCodec::Missing,
17812            InvalidTaskPayloadCodec::Null,
17813            InvalidTaskPayloadCodec::NonString,
17814        ] {
17815            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17816            let client = Client::builder(server.base_url())
17817                .timeout(Duration::from_secs(2))
17818                .build()
17819                .expect("client");
17820            let mut worker = Worker::new(client, "rust-workers")
17821                .worker_id("codec-worker")
17822                .poll_timeout(Duration::from_millis(10));
17823            let handler_calls = Arc::new(AtomicUsize::new(0));
17824
17825            let calls = Arc::clone(&handler_calls);
17826            worker.register_workflow("codec.workflow", move |_ctx, _args| {
17827                calls.fetch_add(1, Ordering::SeqCst);
17828                async move { Ok(Value::Null) }
17829            });
17830            let calls = Arc::clone(&handler_calls);
17831            worker.register_activity("codec.activity", move |_ctx, _args| {
17832                calls.fetch_add(1, Ordering::SeqCst);
17833                async move { Ok(Value::Null) }
17834            });
17835            let calls = Arc::clone(&handler_calls);
17836            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17837                calls.fetch_add(1, Ordering::SeqCst);
17838                async move { Ok(Value::Null) }
17839            });
17840
17841            assert_eq!(
17842                worker.run_once().await.expect("invalid tasks are settled"),
17843                3,
17844                "all {} codec tasks must be handled",
17845                codec_case.label()
17846            );
17847            assert_eq!(
17848                handler_calls.load(Ordering::SeqCst),
17849                0,
17850                "{} task codecs must fail before every handler",
17851                codec_case.label()
17852            );
17853
17854            for path in [
17855                "/api/worker/workflow-tasks/codec-workflow/fail",
17856                "/api/worker/activity-tasks/codec-activity/fail",
17857                "/api/worker/query-tasks/codec-query/fail",
17858            ] {
17859                let body = server.request_body(path);
17860                assert!(
17861                    body["failure"]["message"]
17862                        .as_str()
17863                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
17864                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
17865                    codec_case.label()
17866                );
17867            }
17868            assert_eq!(
17869                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17870                    ["reason"],
17871                "query_payload_decode_failed"
17872            );
17873            for path in [
17874                "/api/worker/workflow-tasks/codec-workflow/complete",
17875                "/api/worker/activity-tasks/codec-activity/complete",
17876                "/api/worker/query-tasks/codec-query/complete",
17877            ] {
17878                assert_eq!(
17879                    server.request_count(path),
17880                    0,
17881                    "invalid {} codec task reached {path}",
17882                    codec_case.label()
17883                );
17884            }
17885        }
17886    }
17887
17888    #[tokio::test]
17889    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17890        let client = Client::new("http://127.0.0.1:8080").expect("client");
17891        let mut worker = Worker::new(client, "rust-workers");
17892        let handler_calls = Arc::new(AtomicUsize::new(0));
17893
17894        let calls = Arc::clone(&handler_calls);
17895        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17896            calls.fetch_add(1, Ordering::SeqCst);
17897            async move { Ok(Value::Null) }
17898        });
17899        let calls = Arc::clone(&handler_calls);
17900        worker.register_activity("codec.activity", move |_ctx, _args| {
17901            calls.fetch_add(1, Ordering::SeqCst);
17902            async move { Ok(Value::Null) }
17903        });
17904        let calls = Arc::clone(&handler_calls);
17905        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17906            calls.fetch_add(1, Ordering::SeqCst);
17907            async move { Ok(Value::Null) }
17908        });
17909        let calls = Arc::clone(&handler_calls);
17910        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17911            calls.fetch_add(1, Ordering::SeqCst);
17912            async move { Ok(Value::Null) }
17913        });
17914
17915        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17916        workflow.payload_codec = "json".to_string();
17917        workflow.arguments = None;
17918        let error = worker
17919            .execute_workflow_task(workflow)
17920            .expect_err("task codec must be checked before workflow invocation");
17921        assert!(error.to_string().contains("unsupported_payload_codec"));
17922
17923        let activity = ActivityTask {
17924            task_id: "activity-invalid-codec".to_string(),
17925            activity_attempt_id: None,
17926            attempt_id: None,
17927            activity_type: "codec.activity".to_string(),
17928            payload_codec: "unknown".to_string(),
17929            arguments: None,
17930            attempt_number: 1,
17931            lease_owner: None,
17932        };
17933        let error = worker
17934            .execute_activity_task(activity)
17935            .await
17936            .expect_err("task codec must be checked before activity invocation");
17937        assert!(error.to_string().contains("unsupported_payload_codec"));
17938
17939        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17940        update.workflow_update_id = Some("update-invalid-codec".to_string());
17941        update.update_name = Some("known".to_string());
17942        update.history_events.push(history_event(
17943            "UpdateAccepted",
17944            json!({
17945                "update_id": "update-invalid-codec",
17946                "update_name": "known",
17947                "arguments": {"codec": "json", "blob": null}
17948            }),
17949        ));
17950        let error = worker
17951            .execute_workflow_task(update)
17952            .expect_err("nested update codec must be checked before handler lookup");
17953        assert!(error.to_string().contains("unsupported_payload_codec"));
17954
17955        let query: QueryTask = serde_json::from_value(json!({
17956            "query_task_id": "query-invalid-codec",
17957            "workflow_type": "codec.workflow",
17958            "query_name": "known",
17959            "payload_codec": DEFAULT_CODEC,
17960            "workflow_arguments": null,
17961            "query_arguments": null,
17962            "history_export": {
17963                "payloads": {"codec": DEFAULT_CODEC},
17964                "signals": [{
17965                    "name": "empty",
17966                    "payload_codec": "json",
17967                    "arguments": null
17968                }]
17969            }
17970        }))
17971        .expect("query task");
17972        let failure = worker
17973            .execute_query_task(query)
17974            .await
17975            .expect_err("exported signal codec must be checked before query invocation");
17976        assert_eq!(failure.reason, "query_payload_decode_failed");
17977        assert!(failure.message.contains("unsupported_payload_codec"));
17978
17979        let exported_history: QueryTask = serde_json::from_value(json!({
17980            "query_task_id": "query-invalid-history-codec",
17981            "workflow_type": "codec.workflow",
17982            "query_name": "known",
17983            "payload_codec": DEFAULT_CODEC,
17984            "history_export": {
17985                "payloads": {"codec": DEFAULT_CODEC},
17986                "history_events": [{
17987                    "type": "ActivityCompleted",
17988                    "payload": {"payload_codec": "unknown", "result": null}
17989                }]
17990            }
17991        }))
17992        .expect("query task");
17993        let failure = worker
17994            .execute_query_task(exported_history)
17995            .await
17996            .expect_err("exported history codec must be checked before query invocation");
17997        assert_eq!(failure.reason, "query_payload_decode_failed");
17998        assert!(failure.message.contains("unsupported_payload_codec"));
17999        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
18000
18001        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
18002        unknown_workflow.arguments = None;
18003        unknown_workflow.history_events.push(history_event(
18004            "SignalReceived",
18005            json!({
18006                "signal_name": "empty",
18007                "payload_codec": "json",
18008                "arguments": null
18009            }),
18010        ));
18011        let error = worker
18012            .execute_workflow_task(unknown_workflow)
18013            .expect_err("history codec must precede unknown workflow outcome");
18014        assert!(error.to_string().contains("unsupported_payload_codec"));
18015
18016        let unknown_activity = ActivityTask {
18017            task_id: "activity-unknown".to_string(),
18018            activity_attempt_id: None,
18019            attempt_id: None,
18020            activity_type: "missing".to_string(),
18021            payload_codec: "json".to_string(),
18022            arguments: None,
18023            attempt_number: 1,
18024            lease_owner: None,
18025        };
18026        let error = worker
18027            .execute_activity_task(unknown_activity)
18028            .await
18029            .expect_err("codec must precede unknown activity outcome");
18030        assert!(error.to_string().contains("unsupported_payload_codec"));
18031
18032        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18033        unknown_update.payload_codec = "json".to_string();
18034        unknown_update.arguments = None;
18035        unknown_update.workflow_update_id = Some("update-unknown".to_string());
18036        unknown_update.update_name = Some("missing".to_string());
18037        let error = worker
18038            .execute_workflow_task(unknown_update)
18039            .expect_err("codec must precede fail_update shortcut");
18040        assert!(error.to_string().contains("unsupported_payload_codec"));
18041
18042        let unknown_query: QueryTask = serde_json::from_value(json!({
18043            "query_task_id": "query-unknown",
18044            "workflow_type": "missing",
18045            "query_name": "missing",
18046            "payload_codec": "json",
18047            "workflow_arguments": null,
18048            "query_arguments": null
18049        }))
18050        .expect("query task");
18051        let failure = worker
18052            .execute_query_task(unknown_query)
18053            .await
18054            .expect_err("codec must precede unknown query outcome");
18055        assert_eq!(failure.reason, "query_payload_decode_failed");
18056        assert!(failure.message.contains("unsupported_payload_codec"));
18057    }
18058
18059    #[tokio::test]
18060    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
18061        let client = Client::new("http://127.0.0.1:8080").expect("client");
18062        let worker = Worker::new(client, "rust-workers");
18063
18064        for event_type in ["SignalReceived", "SignalApplied"] {
18065            for (payload_field, codec) in [
18066                ("value", "json"),
18067                ("input", "unknown"),
18068                ("arguments", "json"),
18069            ] {
18070                let payload = json!({
18071                    "signal_name": "empty",
18072                    payload_field: {"codec": codec, "blob": null}
18073                });
18074                let workflow = workflow_task(
18075                    "missing",
18076                    vec![history_event(event_type, payload.clone())],
18077                    DEFAULT_CODEC,
18078                );
18079                let error = worker
18080                    .execute_workflow_task(workflow)
18081                    .expect_err("signal payload codec must precede unknown workflow outcome");
18082                assert!(
18083                    error.to_string().contains("unsupported_payload_codec"),
18084                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
18085                );
18086
18087                let query: QueryTask = serde_json::from_value(json!({
18088                    "query_task_id": format!("query-{event_type}-{payload_field}"),
18089                    "workflow_type": "missing",
18090                    "query_name": "missing",
18091                    "payload_codec": DEFAULT_CODEC,
18092                    "workflow_arguments": null,
18093                    "query_arguments": null,
18094                    "history_events": [{
18095                        "event_type": event_type,
18096                        "payload": payload
18097                    }]
18098                }))
18099                .expect("query task");
18100                let failure = worker
18101                    .execute_query_task(query)
18102                    .await
18103                    .expect_err("signal payload codec must precede unknown query outcome");
18104                assert_eq!(
18105                    failure.reason, "query_payload_decode_failed",
18106                    "{event_type}.{payload_field} returned an unrelated query outcome"
18107                );
18108                assert!(
18109                    failure.message.contains("unsupported_payload_codec"),
18110                    "{event_type}.{payload_field} returned an unrelated query error: {}",
18111                    failure.message
18112                );
18113            }
18114        }
18115    }
18116
18117    #[test]
18118    fn workflow_context_schedules_activity_until_completion_is_in_history() {
18119        let ctx = WorkflowContext {
18120            state: Arc::new(Mutex::new(
18121                WorkflowState::new_with_identity(
18122                    Vec::new(),
18123                    Some("wf-parent".to_string()),
18124                    Some("run-parent".to_string()),
18125                    "rust-workers".to_string(),
18126                    DEFAULT_CODEC.to_string(),
18127                    None,
18128                )
18129                .expect("workflow state"),
18130            )),
18131        };
18132
18133        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18134        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18135        assert!(matches!(
18136            call.as_mut().poll(&mut task_context),
18137            Poll::Pending
18138        ));
18139
18140        let commands = ctx.take_commands().expect("commands");
18141        assert_eq!(commands[0]["type"], "schedule_activity");
18142        assert_eq!(commands[0]["activity_type"], "hello.activity");
18143    }
18144
18145    #[test]
18146    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18147        let ctx = workflow_context(Vec::new());
18148        let options = ActivityOptions::new()
18149            .task_queue("payments")
18150            .retry_policy(
18151                ActivityRetryPolicy::new(4)
18152                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18153                    .non_retryable_error_type("ValidationError"),
18154            )
18155            .start_to_close_timeout(Duration::from_secs(120))
18156            .schedule_to_start_timeout(Duration::from_secs(10))
18157            .schedule_to_close_timeout(Duration::from_secs(300))
18158            .heartbeat_timeout(Duration::from_secs(15));
18159        let mut call = Box::pin(ctx.activity_with_options(
18160            "charge-card",
18161            options,
18162            json!([{"order_id": "o-1"}]),
18163        ));
18164        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18165
18166        assert!(matches!(
18167            call.as_mut().poll(&mut task_context),
18168            Poll::Pending
18169        ));
18170        assert!(matches!(
18171            call.as_mut().poll(&mut task_context),
18172            Poll::Pending
18173        ));
18174
18175        let commands = ctx.take_commands().expect("activity command");
18176        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18177        assert_eq!(commands[0]["queue"], "payments");
18178        assert_eq!(
18179            commands[0]["retry_policy"],
18180            json!({
18181                "max_attempts": 4,
18182                "backoff_seconds": [1, 3, 9],
18183                "non_retryable_error_types": ["ValidationError"],
18184            })
18185        );
18186        assert_eq!(commands[0]["start_to_close_timeout"], 120);
18187        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18188        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18189        assert_eq!(commands[0]["heartbeat_timeout"], 15);
18190    }
18191
18192    #[test]
18193    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18194        let ctx = workflow_context(Vec::new());
18195        let options = ActivityOptions::new().retry_policy(
18196            ActivityRetryPolicy::new(3)
18197                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18198        );
18199        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18200        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18201
18202        assert!(matches!(
18203            call.as_mut().poll(&mut task_context),
18204            Poll::Pending
18205        ));
18206        assert_eq!(
18207            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18208            json!([1, 2])
18209        );
18210    }
18211
18212    #[test]
18213    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18214        let cases = [
18215            (
18216                ActivityOptions::new().task_queue("  "),
18217                ActivityOptionsErrorKind::EmptyTaskQueue,
18218            ),
18219            (
18220                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18221                ActivityOptionsErrorKind::EmptyRetryPolicy,
18222            ),
18223            (
18224                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18225                ActivityOptionsErrorKind::InvalidMaxAttempts,
18226            ),
18227            (
18228                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18229                    max_attempts: None,
18230                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18231                    non_retryable_error_types: Vec::new(),
18232                }),
18233                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18234            ),
18235            (
18236                ActivityOptions::new().retry_policy(
18237                    ActivityRetryPolicy::new(2)
18238                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18239                ),
18240                ActivityOptionsErrorKind::TooManyBackoffIntervals,
18241            ),
18242            (
18243                ActivityOptions::new().retry_policy(
18244                    ActivityRetryPolicy::new(2).exponential_backoff(
18245                        Duration::from_secs(1),
18246                        0,
18247                        None,
18248                    ),
18249                ),
18250                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18251            ),
18252            (
18253                ActivityOptions::new()
18254                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
18255                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18256            ),
18257            (
18258                ActivityOptions::new().retry_policy(
18259                    ActivityRetryPolicy::new(10_002).exponential_backoff(
18260                        Duration::from_secs(1),
18261                        1,
18262                        None,
18263                    ),
18264                ),
18265                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18266            ),
18267            (
18268                ActivityOptions::new().retry_policy(
18269                    ActivityRetryPolicy::new(2)
18270                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18271                ),
18272                ActivityOptionsErrorKind::BackoffOverflow,
18273            ),
18274        ];
18275
18276        for (options, expected_kind) in cases {
18277            let ctx = workflow_context(Vec::new());
18278            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18279            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18280            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18281                call.as_mut().poll(&mut task_context)
18282            else {
18283                panic!("expected typed activity validation error");
18284            };
18285            assert_eq!(error.kind, expected_kind);
18286            assert!(ctx.take_commands().expect("commands").is_empty());
18287        }
18288    }
18289
18290    #[test]
18291    fn activity_options_validate_positive_and_ordered_timeouts() {
18292        let zero_timeout_cases = [
18293            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18294            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18295            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18296            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18297        ];
18298        for options in zero_timeout_cases {
18299            assert_eq!(
18300                options.validate().expect_err("zero timeout").kind,
18301                ActivityOptionsErrorKind::TimeoutNotPositive
18302            );
18303        }
18304
18305        let ordering_cases = [
18306            ActivityOptions::new()
18307                .heartbeat_timeout(Duration::from_secs(11))
18308                .start_to_close_timeout(Duration::from_secs(10)),
18309            ActivityOptions::new()
18310                .start_to_close_timeout(Duration::from_secs(31))
18311                .schedule_to_close_timeout(Duration::from_secs(30)),
18312            ActivityOptions::new()
18313                .schedule_to_start_timeout(Duration::from_secs(31))
18314                .schedule_to_close_timeout(Duration::from_secs(30)),
18315        ];
18316        for options in ordering_cases {
18317            assert_eq!(
18318                options.validate().expect_err("timeout order").kind,
18319                ActivityOptionsErrorKind::TimeoutOrder
18320            );
18321        }
18322
18323        assert_eq!(
18324            ActivityOptions::new()
18325                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18326                .validate()
18327                .expect_err("protocol integer overflow")
18328                .kind,
18329            ActivityOptionsErrorKind::TimeoutOverflow
18330        );
18331    }
18332
18333    #[test]
18334    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18335        let ctx = workflow_context(completed_retry_activity_history());
18336        let mut call =
18337            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18338        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18339
18340        assert!(matches!(
18341            call.as_mut().poll(&mut task_context),
18342            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18343        ));
18344        assert!(ctx.take_commands().expect("commands").is_empty());
18345        ctx.ensure_history_consumed().expect("history consumed");
18346    }
18347
18348    #[test]
18349    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18350        let mut options = retry_activity_options();
18351        options
18352            .retry_policy
18353            .as_mut()
18354            .expect("retry policy")
18355            .non_retryable_error_types
18356            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18357
18358        let new_ctx = workflow_context(Vec::new());
18359        let mut new_call =
18360            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18361        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18362        assert!(matches!(
18363            new_call.as_mut().poll(&mut task_context),
18364            Poll::Pending
18365        ));
18366        let commands = new_ctx.take_commands().expect("commands");
18367        assert_eq!(commands.len(), 1);
18368        assert_eq!(
18369            commands[0]["retry_policy"]["non_retryable_error_types"],
18370            json!(["PermanentError"])
18371        );
18372
18373        let replay_ctx = workflow_context(completed_retry_activity_history());
18374        let mut replay_call =
18375            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18376        assert!(matches!(
18377            replay_call.as_mut().poll(&mut task_context),
18378            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18379        ));
18380        assert!(replay_ctx.take_commands().expect("commands").is_empty());
18381        replay_ctx
18382            .ensure_history_consumed()
18383            .expect("history consumed");
18384    }
18385
18386    #[test]
18387    fn replayed_intermediate_retry_remains_pending_across_restarts() {
18388        let history = completed_retry_activity_history()
18389            .into_iter()
18390            .take(3)
18391            .collect::<Vec<_>>();
18392
18393        for _restart in 0..2 {
18394            let ctx = workflow_context(history.clone());
18395            let mut call =
18396                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18397            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18398            assert!(matches!(
18399                call.as_mut().poll(&mut task_context),
18400                Poll::Pending
18401            ));
18402            assert!(ctx.take_commands().expect("commands").is_empty());
18403        }
18404    }
18405
18406    #[test]
18407    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18408        let mut changed_queue = retry_activity_options();
18409        changed_queue.task_queue = Some("different-queue".to_string());
18410
18411        let mut changed_max_attempts = retry_activity_options();
18412        let retry_policy = changed_max_attempts
18413            .retry_policy
18414            .as_mut()
18415            .expect("retry policy");
18416        retry_policy.max_attempts = Some(4);
18417
18418        let mut changed_backoff = retry_activity_options();
18419        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18420        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18421            Duration::from_secs(3),
18422            Duration::from_secs(4),
18423        ]));
18424
18425        let mut changed_non_retryable_types = retry_activity_options();
18426        let retry_policy = changed_non_retryable_types
18427            .retry_policy
18428            .as_mut()
18429            .expect("retry policy");
18430        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18431
18432        let mut changed_start_to_close = retry_activity_options();
18433        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18434        let mut changed_schedule_to_start = retry_activity_options();
18435        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18436        let mut changed_schedule_to_close = retry_activity_options();
18437        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18438        let mut changed_heartbeat = retry_activity_options();
18439        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18440
18441        let cases = [
18442            (changed_queue, "activity_task_queue_mismatch"),
18443            (changed_max_attempts, "activity_retry_policy_mismatch"),
18444            (changed_backoff, "activity_retry_policy_mismatch"),
18445            (
18446                changed_non_retryable_types,
18447                "activity_retry_policy_mismatch",
18448            ),
18449            (changed_start_to_close, "activity_retry_policy_mismatch"),
18450            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18451            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18452            (changed_heartbeat, "activity_retry_policy_mismatch"),
18453        ];
18454
18455        for (options, expected_reason) in cases {
18456            let ctx = workflow_context(completed_retry_activity_history());
18457            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18458            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18459            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18460                call.as_mut().poll(&mut task_context)
18461            else {
18462                panic!("changed activity options must fail replay");
18463            };
18464            assert_eq!(failure.reason, expected_reason);
18465            assert_eq!(failure.sequence, Some(1));
18466            assert!(ctx.take_commands().expect("commands").is_empty());
18467        }
18468    }
18469
18470    #[test]
18471    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18472        let cases = [
18473            (
18474                "execution_mode",
18475                json!("local"),
18476                "activity_execution_mode_mismatch",
18477            ),
18478            (
18479                "snapshot_version",
18480                json!(2),
18481                "activity_retry_policy_mismatch",
18482            ),
18483        ];
18484
18485        for (field, value, expected_reason) in cases {
18486            let mut history = completed_retry_activity_history();
18487            let activity = history[0].payload["activity"]
18488                .as_object_mut()
18489                .expect("activity snapshot");
18490            if field == "execution_mode" {
18491                activity.insert(field.to_string(), value);
18492            } else {
18493                activity["retry_policy"]
18494                    .as_object_mut()
18495                    .expect("retry snapshot")
18496                    .insert(field.to_string(), value);
18497            }
18498
18499            let ctx = workflow_context(history);
18500            let mut call =
18501                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18502            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18503            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18504                call.as_mut().poll(&mut task_context)
18505            else {
18506                panic!("changed {field} must fail replay");
18507            };
18508            assert_eq!(failure.reason, expected_reason);
18509            assert_eq!(failure.sequence, Some(1));
18510            assert!(ctx.take_commands().expect("commands").is_empty());
18511        }
18512    }
18513
18514    #[test]
18515    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18516        let mut history = completed_retry_activity_history();
18517        let activity = history[0].payload["activity"]
18518            .as_object_mut()
18519            .expect("activity snapshot");
18520        activity.remove("execution_mode");
18521        activity.remove("retry_policy");
18522
18523        let mut current = retry_activity_options();
18524        current.start_to_close_timeout = Some(Duration::from_secs(45));
18525        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18526        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18527        current.heartbeat_timeout = Some(Duration::from_secs(12));
18528
18529        let ctx = workflow_context(history);
18530        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18531        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18532        assert!(matches!(
18533            call.as_mut().poll(&mut task_context),
18534            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18535        ));
18536        assert!(ctx.take_commands().expect("commands").is_empty());
18537        ctx.ensure_history_consumed().expect("history consumed");
18538    }
18539
18540    #[test]
18541    fn terminal_activity_failed_after_start_returns_typed_failure() {
18542        let history = vec![
18543            history_event(
18544                "ActivityScheduled",
18545                json!({
18546                    "sequence": 1,
18547                    "activity_type": "flaky",
18548                    "activity_execution_id": "act-terminal",
18549                    "activity": {
18550                        "id": "act-terminal",
18551                        "sequence": 1,
18552                        "type": "flaky",
18553                        "queue": "critical-activities",
18554                        "retry_policy": {
18555                            "snapshot_version": 1,
18556                            "max_attempts": 3,
18557                            "backoff_seconds": [2, 4],
18558                            "non_retryable_error_types": ["PermanentError"]
18559                        }
18560                    }
18561                }),
18562            ),
18563            history_event(
18564                "ActivityStarted",
18565                json!({
18566                    "sequence": 1,
18567                    "activity_type": "flaky",
18568                    "activity_execution_id": "act-terminal",
18569                    "activity_attempt_id": "attempt-1",
18570                    "attempt_number": 1
18571                }),
18572            ),
18573            history_event(
18574                "ActivityFailed",
18575                json!({
18576                    "sequence": 1,
18577                    "activity_type": "flaky",
18578                    "activity_execution_id": "act-terminal",
18579                    "activity_attempt_id": "attempt-1",
18580                    "attempt_number": 1,
18581                    "failure_id": "failure-terminal",
18582                    "failure_category": "activity",
18583                    "exception_type": "PermanentError",
18584                    "message": "cannot retry",
18585                    "non_retryable": true
18586                }),
18587            ),
18588        ];
18589        let ctx = workflow_context(history);
18590        let mut call =
18591            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18592        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18593
18594        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18595            call.as_mut().poll(&mut task_context)
18596        else {
18597            panic!("terminal ActivityFailed must settle the activity future");
18598        };
18599        assert_eq!(failure.kind, ActivityFailureKind::Failed);
18600        assert_eq!(
18601            failure.activity_execution_id.as_deref(),
18602            Some("act-terminal")
18603        );
18604        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18605        assert!(failure.non_retryable);
18606        assert!(ctx.take_commands().expect("commands").is_empty());
18607        ctx.ensure_history_consumed().expect("history consumed");
18608    }
18609
18610    #[test]
18611    fn activity_terminal_events_return_machine_readable_failures() {
18612        let cases = [
18613            (
18614                "ActivityFailed",
18615                json!({
18616                    "sequence": 1,
18617                    "activity_type": "charge-card",
18618                    "activity_execution_id": "act-1",
18619                    "activity_attempt_id": "attempt-2",
18620                    "attempt_number": 2,
18621                    "failure_id": "failure-1",
18622                    "failure_category": "activity",
18623                    "exception_type": "PaymentDeclined",
18624                    "exception_class": "payments.PaymentDeclined",
18625                    "message": "card declined",
18626                    "non_retryable": true
18627                }),
18628                ActivityFailureKind::Failed,
18629                "activity",
18630            ),
18631            (
18632                "ActivityCancelled",
18633                json!({
18634                    "sequence": 1,
18635                    "activity_type": "charge-card",
18636                    "activity_execution_id": "act-1",
18637                    "activity_attempt_id": "attempt-1"
18638                }),
18639                ActivityFailureKind::Cancelled,
18640                "cancelled",
18641            ),
18642        ];
18643
18644        for (event_type, payload, expected_kind, expected_reason) in cases {
18645            let ctx = workflow_context(vec![history_event(event_type, payload)]);
18646            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18647            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18648            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18649                call.as_mut().poll(&mut task_context)
18650            else {
18651                panic!("expected terminal activity failure");
18652            };
18653            assert_eq!(failure.kind, expected_kind);
18654            assert_eq!(failure.reason, expected_reason);
18655            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18656            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18657        }
18658    }
18659
18660    #[test]
18661    fn every_activity_timeout_class_is_typed() {
18662        for timeout_kind in [
18663            "start_to_close",
18664            "schedule_to_start",
18665            "schedule_to_close",
18666            "heartbeat",
18667        ] {
18668            let ctx = workflow_context(vec![history_event(
18669                "ActivityTimedOut",
18670                json!({
18671                    "sequence": 1,
18672                    "activity_type": "slow",
18673                    "activity_execution_id": "act-timeout",
18674                    "activity_attempt_id": "attempt-timeout",
18675                    "failure_category": "timeout",
18676                    "timeout_kind": timeout_kind,
18677                    "message": "deadline expired"
18678                }),
18679            )]);
18680            let mut call = Box::pin(ctx.activity("slow", json!([])));
18681            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18682            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18683                call.as_mut().poll(&mut task_context)
18684            else {
18685                panic!("expected timeout failure");
18686            };
18687            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18688            assert_eq!(failure.reason, timeout_kind);
18689            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18690            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18691        }
18692    }
18693
18694    #[test]
18695    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18696        let ctx = workflow_context(Vec::new());
18697        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18698        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18699
18700        assert!(matches!(
18701            sleep.as_mut().poll(&mut task_context),
18702            Poll::Pending
18703        ));
18704        assert!(matches!(
18705            sleep.as_mut().poll(&mut task_context),
18706            Poll::Pending
18707        ));
18708
18709        let commands = ctx.take_commands().expect("timer command");
18710        assert_eq!(
18711            commands,
18712            vec![json!({
18713                "type": "start_timer",
18714                "delay_seconds": 2,
18715            })]
18716        );
18717    }
18718
18719    #[test]
18720    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18721        let history = vec![
18722            history_event(
18723                "TimerScheduled",
18724                json!({
18725                    "sequence": 1,
18726                    "timer_id": "timer-1",
18727                    "delay_seconds": 5,
18728                    "fire_at": "2026-07-11T12:00:05Z",
18729                }),
18730            ),
18731            history_event(
18732                "TimerFired",
18733                json!({
18734                    "sequence": 1,
18735                    "timer_id": "timer-1",
18736                    "delay_seconds": 5,
18737                    "fire_at": "2026-07-11T12:00:05Z",
18738                    "fired_at": "2026-07-11T12:00:05Z",
18739                }),
18740            ),
18741        ];
18742
18743        for _restart in 0..2 {
18744            let ctx = workflow_context(history.clone());
18745            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18746            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18747            assert!(matches!(
18748                sleep.as_mut().poll(&mut task_context),
18749                Poll::Ready(Ok(()))
18750            ));
18751            assert!(ctx.take_commands().expect("commands").is_empty());
18752            ctx.ensure_history_consumed().expect("history consumed");
18753        }
18754    }
18755
18756    #[test]
18757    fn workflow_sleep_rejects_changed_delay_during_replay() {
18758        let ctx = workflow_context(vec![
18759            history_event(
18760                "TimerScheduled",
18761                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18762            ),
18763            history_event(
18764                "TimerFired",
18765                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18766            ),
18767        ]);
18768        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18769        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18770
18771        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18772            sleep.as_mut().poll(&mut task_context)
18773        else {
18774            panic!("changed timer delay must be rejected");
18775        };
18776        assert_eq!(failure.reason, "timer_delay_mismatch");
18777        assert_eq!(failure.sequence, Some(1));
18778    }
18779
18780    #[test]
18781    fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18782        let ctx = workflow_context(Vec::new());
18783        let mut wait = Box::pin(
18784            ctx.wait_condition(
18785                ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18786                    .timeout(Duration::from_millis(60_001)),
18787                || Ok(false),
18788            ),
18789        );
18790        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18791
18792        assert!(matches!(
18793            wait.as_mut().poll(&mut task_context),
18794            Poll::Pending
18795        ));
18796        assert!(matches!(
18797            wait.as_mut().poll(&mut task_context),
18798            Poll::Pending
18799        ));
18800        assert_eq!(
18801            ctx.take_commands().expect("condition command"),
18802            vec![json!({
18803                "type": "open_condition_wait",
18804                "condition_wait_occurrence_id": "rust:condition-wait:0",
18805                "condition_key": "approval.ready",
18806                "condition_definition_fingerprint": "sha256:approval-v1",
18807                "timeout_seconds": 61,
18808            })]
18809        );
18810    }
18811
18812    #[test]
18813    fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18814        let ctx = workflow_context(Vec::new());
18815        let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18816        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18817        assert!(matches!(
18818            satisfied.as_mut().poll(&mut task_context),
18819            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18820        ));
18821
18822        let mut timed_out = Box::pin(wait_condition!(
18823            ctx,
18824            "no-wait",
18825            timeout: Duration::ZERO,
18826            || Ok(false),
18827        ));
18828        assert!(matches!(
18829            timed_out.as_mut().poll(&mut task_context),
18830            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18831        ));
18832        assert!(ctx.take_commands().expect("commands").is_empty());
18833    }
18834
18835    #[test]
18836    fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18837        let signal_history = vec![
18838            history_event(
18839                "ConditionWaitOpened",
18840                json!({
18841                    "sequence": 4,
18842                    "condition_wait_id": "condition:4",
18843                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18844                    "condition_key": "approval",
18845                    "condition_definition_fingerprint": "sha256:approval-v1",
18846                    "timeout_seconds": 30,
18847                }),
18848            ),
18849            history_event(
18850                "SignalReceived",
18851                json!({
18852                    "workflow_sequence": 4,
18853                    "signal_name": "approve",
18854                    "arguments": fixture_envelope(json!(["Ada"])),
18855                }),
18856            ),
18857        ];
18858        for _worker_before_or_after_restart in 0..2 {
18859            let ctx = workflow_context(signal_history.clone());
18860            let predicate_ctx = ctx.clone();
18861            let mut wait = Box::pin(
18862                ctx.wait_condition(
18863                    ConditionWaitOptions::new("approval", "sha256:approval-v1")
18864                        .timeout(Duration::from_secs(30)),
18865                    move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18866                ),
18867            );
18868            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18869            assert!(matches!(
18870                wait.as_mut().poll(&mut task_context),
18871                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18872            ));
18873            assert!(ctx.take_commands().expect("commands").is_empty());
18874            ctx.ensure_history_consumed().expect("condition consumed");
18875        }
18876
18877        let update_history = vec![
18878            history_event(
18879                "ConditionWaitOpened",
18880                json!({
18881                    "sequence": 7,
18882                    "condition_wait_id": "condition:7",
18883                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18884                    "condition_key": "update-approval",
18885                    "condition_definition_fingerprint": "sha256:update-approval-v1",
18886                }),
18887            ),
18888            history_event(
18889                "UpdateApplied",
18890                json!({
18891                    "sequence": 7,
18892                    "update_id": "update-1",
18893                    "update_name": "approve",
18894                    "arguments": fixture_envelope(json!([true])),
18895                }),
18896            ),
18897        ];
18898        let ctx = workflow_context(update_history);
18899        let predicate_ctx = ctx.clone();
18900        let mut wait = Box::pin(ctx.wait_condition(
18901            ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18902            move || {
18903                Ok(predicate_ctx
18904                    .updates("approve")?
18905                    .first()
18906                    .and_then(|arguments| arguments.first())
18907                    .and_then(Value::as_bool)
18908                    == Some(true))
18909            },
18910        ));
18911        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18912        assert!(matches!(
18913            wait.as_mut().poll(&mut task_context),
18914            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18915        ));
18916        assert!(ctx.take_commands().expect("commands").is_empty());
18917        ctx.ensure_history_consumed().expect("condition consumed");
18918    }
18919
18920    #[test]
18921    fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18922        let open_history = vec![
18923            history_event(
18924                "ConditionWaitOpened",
18925                json!({
18926                    "sequence": 3,
18927                    "condition_wait_id": "condition:3",
18928                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18929                    "condition_key": "two-votes",
18930                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18931                    "timeout_seconds": 120,
18932                }),
18933            ),
18934            history_event(
18935                "SignalReceived",
18936                json!({
18937                    "workflow_sequence": 3,
18938                    "signal_name": "vote",
18939                    "arguments": fixture_envelope(json!(["first"])),
18940                }),
18941            ),
18942        ];
18943        for _worker_before_or_after_restart in 0..2 {
18944            let ctx = workflow_context(open_history.clone());
18945            let predicate_ctx = ctx.clone();
18946            let mut wait = Box::pin(
18947                ctx.wait_condition(
18948                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18949                        .timeout(Duration::from_secs(120)),
18950                    move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18951                ),
18952            );
18953            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18954            assert!(matches!(
18955                wait.as_mut().poll(&mut task_context),
18956                Poll::Pending
18957            ));
18958            assert_eq!(
18959                ctx.take_commands().expect("reopened condition"),
18960                vec![json!({
18961                    "type": "open_condition_wait",
18962                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18963                    "condition_key": "two-votes",
18964                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18965                    "timeout_seconds": 120,
18966                })]
18967            );
18968        }
18969
18970        let satisfied_ctx = workflow_context(vec![
18971            history_event(
18972                "ConditionWaitOpened",
18973                json!({
18974                    "sequence": 5,
18975                    "condition_wait_id": "condition:5",
18976                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18977                    "condition_key": "approval",
18978                    "condition_definition_fingerprint": "sha256:approval-v1",
18979                }),
18980            ),
18981            history_event(
18982                "ConditionWaitSatisfied",
18983                json!({
18984                    "sequence": 5,
18985                    "condition_wait_id": "condition:5",
18986                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18987                    "condition_key": "approval",
18988                    "condition_definition_fingerprint": "sha256:approval-v1",
18989                }),
18990            ),
18991        ]);
18992        let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18993            ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18994            || Ok(false),
18995        ));
18996        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18997        assert!(matches!(
18998            satisfied.as_mut().poll(&mut task_context),
18999            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19000        ));
19001
19002        let timed_out_ctx = workflow_context(vec![
19003            history_event(
19004                "ConditionWaitOpened",
19005                json!({
19006                    "sequence": 8,
19007                    "condition_wait_id": "condition:8",
19008                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19009                    "condition_key": "approval-timeout",
19010                    "condition_definition_fingerprint": "sha256:approval-timeout-v1",
19011                    "timeout_seconds": 5,
19012                }),
19013            ),
19014            history_event(
19015                "TimerScheduled",
19016                json!({
19017                    "sequence": 9,
19018                    "timer_id": "condition-timer:9",
19019                    "timer_kind": "condition_timeout",
19020                    "condition_wait_id": "condition:8",
19021                    "delay_seconds": 5,
19022                }),
19023            ),
19024            history_event(
19025                "TimerFired",
19026                json!({
19027                    "sequence": 9,
19028                    "timer_id": "condition-timer:9",
19029                    "timer_kind": "condition_timeout",
19030                    "condition_wait_id": "condition:8",
19031                    "delay_seconds": 5,
19032                }),
19033            ),
19034        ]);
19035        let mut timed_out = Box::pin(
19036            timed_out_ctx.wait_condition(
19037                ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
19038                    .timeout(Duration::from_secs(5)),
19039                || Ok(true),
19040            ),
19041        );
19042        assert!(matches!(
19043            timed_out.as_mut().poll(&mut task_context),
19044            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
19045        ));
19046    }
19047
19048    #[test]
19049    fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
19050        let history = vec![
19051            history_event(
19052                "ConditionWaitOpened",
19053                json!({
19054                    "sequence": 3,
19055                    "condition_wait_id": "condition:3",
19056                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19057                    "condition_key": "two-votes",
19058                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19059                }),
19060            ),
19061            history_event(
19062                "SignalReceived",
19063                json!({
19064                    "workflow_sequence": 3,
19065                    "signal_name": "vote",
19066                    "arguments": fixture_envelope(json!(["first"])),
19067                }),
19068            ),
19069            history_event(
19070                "ConditionWaitSatisfied",
19071                json!({
19072                    "sequence": 3,
19073                    "condition_wait_id": "condition:3",
19074                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19075                    "condition_key": "two-votes",
19076                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19077                }),
19078            ),
19079            history_event(
19080                "ConditionWaitOpened",
19081                json!({
19082                    "sequence": 5,
19083                    "condition_wait_id": "condition:5",
19084                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19085                    "condition_key": "two-votes",
19086                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19087                }),
19088            ),
19089            history_event(
19090                "SignalReceived",
19091                json!({
19092                    "workflow_sequence": 5,
19093                    "signal_name": "vote",
19094                    "arguments": fixture_envelope(json!(["second"])),
19095                }),
19096            ),
19097            history_event(
19098                "ConditionWaitSatisfied",
19099                json!({
19100                    "sequence": 5,
19101                    "condition_wait_id": "condition:5",
19102                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19103                    "condition_key": "two-votes",
19104                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19105                }),
19106            ),
19107        ];
19108        for _cold_worker_or_restart in 0..2 {
19109            let ctx = workflow_context(history.clone());
19110            let predicate_ctx = ctx.clone();
19111            let mut wait = Box::pin(ctx.wait_condition(
19112                ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19113                move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19114            ));
19115            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19116
19117            assert!(matches!(
19118                wait.as_mut().poll(&mut task_context),
19119                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19120            ));
19121            assert!(ctx.take_commands().expect("commands").is_empty());
19122            ctx.ensure_history_consumed()
19123                .expect("every physical wait-open is consumed");
19124        }
19125    }
19126
19127    #[test]
19128    fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19129        let history = vec![
19130            history_event(
19131                "ConditionWaitOpened",
19132                json!({
19133                    "sequence": 3,
19134                    "condition_wait_id": "condition:3",
19135                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19136                    "condition_key": "approved",
19137                    "condition_definition_fingerprint": "sha256:approved-v1",
19138                }),
19139            ),
19140            history_event(
19141                "UpdateApplied",
19142                json!({
19143                    "sequence": 3,
19144                    "update_id": "update-1",
19145                    "update_name": "approve",
19146                    "arguments": fixture_envelope(json!([false])),
19147                }),
19148            ),
19149            history_event(
19150                "ConditionWaitOpened",
19151                json!({
19152                    "sequence": 5,
19153                    "condition_wait_id": "condition:5",
19154                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19155                    "condition_key": "approved",
19156                    "condition_definition_fingerprint": "sha256:approved-v1",
19157                }),
19158            ),
19159            history_event(
19160                "UpdateApplied",
19161                json!({
19162                    "sequence": 5,
19163                    "update_id": "update-2",
19164                    "update_name": "approve",
19165                    "arguments": fixture_envelope(json!([true])),
19166                }),
19167            ),
19168        ];
19169
19170        for _cold_worker_or_restart in 0..2 {
19171            let ctx = workflow_context(history.clone());
19172            let predicate_ctx = ctx.clone();
19173            let mut wait = Box::pin(ctx.wait_condition(
19174                ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19175                move || {
19176                    Ok(predicate_ctx
19177                        .updates("approve")?
19178                        .last()
19179                        .and_then(|arguments| arguments.first())
19180                        .and_then(Value::as_bool)
19181                        == Some(true))
19182                },
19183            ));
19184            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19185
19186            assert!(matches!(
19187                wait.as_mut().poll(&mut task_context),
19188                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19189            ));
19190            assert!(ctx.take_commands().expect("commands").is_empty());
19191            ctx.ensure_history_consumed()
19192                .expect("every update-driven reopen is consumed");
19193        }
19194    }
19195
19196    #[test]
19197    fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19198        for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19199            ("shared", "sha256:first", "shared", "sha256:second"),
19200            ("first", "sha256:shared", "second", "sha256:shared"),
19201            ("shared", "sha256:shared", "shared", "sha256:shared"),
19202            ("first", "sha256:first", "second", "sha256:second"),
19203        ] {
19204            let history = vec![
19205                history_event(
19206                    "ConditionWaitOpened",
19207                    json!({
19208                        "sequence": 3,
19209                        "condition_wait_id": "condition:3",
19210                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19211                        "condition_key": first_key,
19212                        "condition_definition_fingerprint": first_fingerprint,
19213                    }),
19214                ),
19215                history_event(
19216                    "ConditionWaitSatisfied",
19217                    json!({
19218                        "sequence": 3,
19219                        "condition_wait_id": "condition:3",
19220                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19221                        "condition_key": first_key,
19222                        "condition_definition_fingerprint": first_fingerprint,
19223                    }),
19224                ),
19225                history_event(
19226                    "ConditionWaitOpened",
19227                    json!({
19228                        "sequence": 4,
19229                        "condition_wait_id": "condition:4",
19230                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19231                        "condition_key": second_key,
19232                        "condition_definition_fingerprint": second_fingerprint,
19233                    }),
19234                ),
19235                history_event(
19236                    "ConditionWaitSatisfied",
19237                    json!({
19238                        "sequence": 4,
19239                        "condition_wait_id": "condition:4",
19240                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19241                        "condition_key": second_key,
19242                        "condition_definition_fingerprint": second_fingerprint,
19243                    }),
19244                ),
19245            ];
19246            for _cold_worker_or_restart in 0..2 {
19247                let ctx = workflow_context(history.clone());
19248                let mut task_context = TaskContext::from_waker(noop_waker_ref());
19249                let mut first = Box::pin(ctx.wait_condition(
19250                    ConditionWaitOptions::new(first_key, first_fingerprint),
19251                    || Ok(false),
19252                ));
19253                assert!(matches!(
19254                    first.as_mut().poll(&mut task_context),
19255                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19256                ));
19257
19258                let mut second = Box::pin(ctx.wait_condition(
19259                    ConditionWaitOptions::new(second_key, second_fingerprint),
19260                    || Ok(false),
19261                ));
19262                assert!(matches!(
19263                    second.as_mut().poll(&mut task_context),
19264                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19265                ));
19266                assert!(ctx.take_commands().expect("commands").is_empty());
19267                ctx.ensure_history_consumed()
19268                    .expect("each authored wait consumes one occurrence");
19269            }
19270        }
19271    }
19272
19273    #[test]
19274    fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19275        fn worker() -> Worker {
19276            let client = Client::new("http://127.0.0.1:8080").expect("client");
19277            let mut worker = Worker::new(client, "rust-workers");
19278            worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19279                let mut outcomes = Vec::new();
19280                for _ in 0..2 {
19281                    outcomes.push(
19282                        ctx.wait_condition(
19283                            ConditionWaitOptions::new("shared", "sha256:shared"),
19284                            || Ok(false),
19285                        )
19286                        .await?,
19287                    );
19288                }
19289                Ok(json!(outcomes))
19290            });
19291            worker
19292        }
19293
19294        let task = workflow_task(
19295            "rust.condition-loop",
19296            vec![
19297                history_event(
19298                    "ConditionWaitOpened",
19299                    json!({
19300                        "sequence": 1,
19301                        "condition_wait_id": "condition:1",
19302                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19303                        "condition_key": "shared",
19304                        "condition_definition_fingerprint": "sha256:shared",
19305                    }),
19306                ),
19307                history_event(
19308                    "ConditionWaitSatisfied",
19309                    json!({
19310                        "sequence": 1,
19311                        "condition_wait_id": "condition:1",
19312                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19313                        "condition_key": "shared",
19314                        "condition_definition_fingerprint": "sha256:shared",
19315                    }),
19316                ),
19317                history_event(
19318                    "ConditionWaitOpened",
19319                    json!({
19320                        "sequence": 2,
19321                        "condition_wait_id": "condition:2",
19322                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19323                        "condition_key": "shared",
19324                        "condition_definition_fingerprint": "sha256:shared",
19325                    }),
19326                ),
19327                history_event(
19328                    "ConditionWaitSatisfied",
19329                    json!({
19330                        "sequence": 2,
19331                        "condition_wait_id": "condition:2",
19332                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19333                        "condition_key": "shared",
19334                        "condition_definition_fingerprint": "sha256:shared",
19335                    }),
19336                ),
19337            ],
19338            DEFAULT_CODEC,
19339        );
19340
19341        for _cold_worker_or_restart in 0..2 {
19342            let commands = worker()
19343                .execute_workflow_task(task.clone())
19344                .expect("adjacent loop waits replay deterministically");
19345            assert_eq!(commands.len(), 1);
19346            assert_eq!(commands[0]["type"], "complete_workflow");
19347            assert_eq!(
19348                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19349                json!(["satisfied", "satisfied"])
19350            );
19351        }
19352    }
19353
19354    #[test]
19355    fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19356        let history = vec![history_event(
19357            "ConditionWaitOpened",
19358            json!({
19359                "sequence": 12,
19360                "condition_wait_id": "condition:12",
19361                "condition_wait_occurrence_id": "rust:condition-wait:0",
19362                "condition_key": "approval",
19363                "condition_definition_fingerprint": "sha256:approval-v1",
19364                "timeout_seconds": 30,
19365            }),
19366        )];
19367        for (options, expected_reason) in [
19368            (
19369                ConditionWaitOptions::new("changed", "sha256:approval-v1")
19370                    .timeout(Duration::from_secs(30)),
19371                "condition_wait_key_mismatch",
19372            ),
19373            (
19374                ConditionWaitOptions::new("approval", "sha256:approval-v2")
19375                    .timeout(Duration::from_secs(30)),
19376                "condition_wait_predicate_mismatch",
19377            ),
19378            (
19379                ConditionWaitOptions::new("approval", "sha256:approval-v1")
19380                    .timeout(Duration::from_secs(29)),
19381                "condition_wait_timeout_mismatch",
19382            ),
19383        ] {
19384            let ctx = workflow_context(history.clone());
19385            let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19386            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19387            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19388                wait.as_mut().poll(&mut task_context)
19389            else {
19390                panic!("changed condition definition must fail replay");
19391            };
19392            assert_eq!(failure.reason, expected_reason);
19393            assert_eq!(failure.sequence, Some(12));
19394        }
19395    }
19396
19397    #[test]
19398    fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19399        let error = WorkflowState::new(
19400            vec![history_event(
19401                "ConditionWaitOpened",
19402                json!({
19403                    "sequence": 12,
19404                    "condition_wait_id": "condition:12",
19405                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19406                    "condition_key": "approval",
19407                }),
19408            )],
19409            "rust-workers".to_string(),
19410            DEFAULT_CODEC.to_string(),
19411            None,
19412        )
19413        .expect_err("condition history without a predicate fingerprint must fail");
19414
19415        assert!(matches!(
19416            error,
19417            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19418                if reason == "condition_wait_predicate_fingerprint_missing"
19419        ));
19420    }
19421
19422    #[test]
19423    fn condition_wait_history_requires_authored_occurrence_identity() {
19424        let error = WorkflowState::new(
19425            vec![history_event(
19426                "ConditionWaitOpened",
19427                json!({
19428                    "sequence": 12,
19429                    "condition_wait_id": "condition:12",
19430                    "condition_key": "approval",
19431                    "condition_definition_fingerprint": "sha256:approval-v1",
19432                }),
19433            )],
19434            "rust-workers".to_string(),
19435            DEFAULT_CODEC.to_string(),
19436            None,
19437        )
19438        .expect_err("condition history without occurrence identity must fail");
19439
19440        assert!(matches!(
19441            error,
19442            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19443                if reason == "condition_wait_occurrence_id_missing"
19444        ));
19445    }
19446
19447    #[test]
19448    fn typed_search_attribute_updates_validate_emit_and_replay() {
19449        let update = SearchAttributeUpdate::new()
19450            .keyword("OrderStatus", " waiting ")
19451            .expect("keyword")
19452            .int("Attempt", 3)
19453            .expect("int")
19454            .bool("Escalated", false)
19455            .expect("bool")
19456            .keyword_list("Regions", ["us-east", "eu-west"])
19457            .expect("list")
19458            .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19459            .expect("datetime")
19460            .delete("LegacyStatus")
19461            .expect("delete");
19462        let ctx = workflow_context(Vec::new());
19463        ctx.upsert_search_attributes(update.clone())
19464            .expect("typed update");
19465        assert_eq!(
19466            ctx.take_commands().expect("search-attribute command"),
19467            vec![json!({
19468                "type": "upsert_search_attributes",
19469                "attributes": {
19470                    "Attempt": 3,
19471                    "Escalated": false,
19472                    "LegacyStatus": null,
19473                    "OrderStatus": "waiting",
19474                    "Regions": ["us-east", "eu-west"],
19475                    "UpdatedAt": "2026-08-22T04:00:00Z",
19476                },
19477                "attribute_types": {
19478                    "Attempt": "int",
19479                    "Escalated": "bool",
19480                    "OrderStatus": "keyword",
19481                    "Regions": "keyword_list",
19482                    "UpdatedAt": "datetime",
19483                },
19484            })]
19485        );
19486
19487        let replay = workflow_context(vec![history_event(
19488            "SearchAttributesUpserted",
19489            json!({
19490                "sequence": 6,
19491                "attributes": {
19492                    "Attempt": 3,
19493                    "Escalated": false,
19494                    "LegacyStatus": null,
19495                    "OrderStatus": "waiting",
19496                    "Regions": ["us-east", "eu-west"],
19497                    "UpdatedAt": "2026-08-22T04:00:00Z",
19498                },
19499                "attribute_types": {
19500                    "Attempt": "int",
19501                    "Escalated": "bool",
19502                    "OrderStatus": "keyword",
19503                    "Regions": "keyword_list",
19504                    "UpdatedAt": "datetime",
19505                },
19506                "merged": {},
19507            }),
19508        )]);
19509        replay
19510            .upsert_search_attributes(update)
19511            .expect("matching update replays");
19512        assert!(replay.take_commands().expect("commands").is_empty());
19513        replay.ensure_history_consumed().expect("history consumed");
19514
19515        let type_drift = workflow_context(vec![history_event(
19516            "SearchAttributesUpserted",
19517            json!({
19518                "sequence": 7,
19519                "attributes": {"OrderStatus": "waiting"},
19520                "attribute_types": {"OrderStatus": "keyword"},
19521                "merged": {"OrderStatus": "waiting"},
19522            }),
19523        )]);
19524        let error = type_drift
19525            .upsert_search_attributes(
19526                SearchAttributeUpdate::new()
19527                    .string("OrderStatus", "waiting")
19528                    .expect("string update"),
19529            )
19530            .expect_err("same JSON value with a changed type must fail replay");
19531        assert!(matches!(
19532            error,
19533            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19534                if reason == "search_attribute_type_mismatch"
19535        ));
19536
19537        let malformed_types = WorkflowState::new(
19538            vec![history_event(
19539                "SearchAttributesUpserted",
19540                json!({
19541                    "sequence": 8,
19542                    "attributes": {"OrderStatus": "waiting"},
19543                    "attribute_types": {"OrderStatus": "unsupported"},
19544                    "merged": {"OrderStatus": "waiting"},
19545                }),
19546            )],
19547            "rust-workers".to_string(),
19548            DEFAULT_CODEC.to_string(),
19549            None,
19550        )
19551        .expect_err("unsupported search-attribute type metadata must fail");
19552        assert!(matches!(
19553            malformed_types,
19554            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19555                if reason == "search_attribute_types_malformed"
19556        ));
19557
19558        assert!(matches!(
19559            SearchAttributeUpdate::new().keyword("bad key", "value"),
19560            Err(SearchAttributeUpdateError::InvalidKey(_))
19561        ));
19562        assert!(matches!(
19563            SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19564            Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19565        ));
19566        assert!(matches!(
19567            SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19568            Err(SearchAttributeUpdateError::ValueTooLong { .. })
19569        ));
19570        assert!(matches!(
19571            SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19572            Err(SearchAttributeUpdateError::InvalidDateTime(_))
19573        ));
19574        assert!(matches!(
19575            workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19576            Err(Error::InvalidSearchAttributeUpdate(
19577                SearchAttributeUpdateError::Empty
19578            ))
19579        ));
19580    }
19581
19582    #[test]
19583    fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19584        let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19585        let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19586
19587        assert!(SearchAttributeUpdate::new()
19588            .string("AsciiDescription", ascii)
19589            .is_ok());
19590        assert!(SearchAttributeUpdate::new()
19591            .string("Utf8Description", utf8)
19592            .is_ok());
19593        assert!(matches!(
19594            SearchAttributeUpdate::new().string(
19595                "TooLongDescription",
19596                "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19597            ),
19598            Err(SearchAttributeUpdateError::ValueTooLong {
19599                kind: "string",
19600                limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19601                ..
19602            })
19603        ));
19604    }
19605
19606    #[test]
19607    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19608        let lone_fire = WorkflowState::new(
19609            vec![history_event(
19610                "TimerFired",
19611                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19612            )],
19613            "rust-workers".to_string(),
19614            DEFAULT_CODEC.to_string(),
19615            None,
19616        )
19617        .expect_err("TimerFired requires TimerScheduled");
19618        assert!(matches!(
19619            lone_fire,
19620            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19621                if reason == "timer_schedule_missing_or_duplicate"
19622        ));
19623
19624        let wrong_identity = WorkflowState::new(
19625            vec![
19626                history_event(
19627                    "TimerScheduled",
19628                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19629                ),
19630                history_event(
19631                    "TimerFired",
19632                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19633                ),
19634            ],
19635            "rust-workers".to_string(),
19636            DEFAULT_CODEC.to_string(),
19637            None,
19638        )
19639        .expect_err("fire must match scheduled timer identity");
19640        assert!(matches!(
19641            wrong_identity,
19642            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19643                if reason == "timer_identity_mismatch"
19644        ));
19645
19646        let duplicate_fire = WorkflowState::new(
19647            vec![
19648                history_event(
19649                    "TimerScheduled",
19650                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19651                ),
19652                history_event(
19653                    "TimerFired",
19654                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19655                ),
19656                history_event(
19657                    "TimerFired",
19658                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19659                ),
19660            ],
19661            "rust-workers".to_string(),
19662            DEFAULT_CODEC.to_string(),
19663            None,
19664        )
19665        .expect_err("a durable timer cannot fire twice");
19666        assert!(matches!(
19667            duplicate_fire,
19668            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19669                if reason == "duplicate_timer_fire"
19670        ));
19671
19672        let wrong_fired_delay = WorkflowState::new(
19673            vec![
19674                history_event(
19675                    "TimerScheduled",
19676                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19677                ),
19678                history_event(
19679                    "TimerFired",
19680                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19681                ),
19682            ],
19683            "rust-workers".to_string(),
19684            DEFAULT_CODEC.to_string(),
19685            None,
19686        )
19687        .expect_err("timer schedule and fire delays must agree");
19688        assert!(matches!(
19689            wrong_fired_delay,
19690            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19691                if reason == "timer_history_delay_mismatch"
19692        ));
19693    }
19694
19695    #[test]
19696    fn replay_rejects_activity_moved_before_recorded_timer() {
19697        let ctx = workflow_context(vec![
19698            history_event(
19699                "TimerScheduled",
19700                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19701            ),
19702            history_event(
19703                "TimerFired",
19704                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19705            ),
19706            history_event(
19707                "ActivityCompleted",
19708                json!({
19709                    "sequence": 2,
19710                    "activity_type": "after-timer",
19711                    "payload_codec": DEFAULT_CODEC,
19712                    "result": fixture_envelope(json!("done")),
19713                }),
19714            ),
19715        ]);
19716        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19717        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19718
19719        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19720            activity.as_mut().poll(&mut task_context)
19721        else {
19722            panic!("reordered durable command must be rejected");
19723        };
19724        assert_eq!(failure.reason, "recorded_command_mismatch");
19725        assert_eq!(failure.sequence, Some(1));
19726        assert_eq!(failure.expected.as_deref(), Some("timer"));
19727        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19728    }
19729
19730    #[test]
19731    fn workflow_context_emits_a_typed_named_signal_wait() {
19732        let ctx = workflow_context(Vec::new());
19733        let mut signal = Box::pin(ctx.wait_signal("finish"));
19734        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19735
19736        assert!(matches!(
19737            signal.as_mut().poll(&mut task_context),
19738            Poll::Pending
19739        ));
19740        assert_eq!(
19741            ctx.take_commands().expect("signal-wait command"),
19742            vec![json!({
19743                "type": "open_signal_wait",
19744                "signal_name": "finish",
19745            })]
19746        );
19747    }
19748
19749    #[test]
19750    fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19751        let ctx = workflow_context(Vec::new());
19752        let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19753        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19754
19755        let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19756        else {
19757            panic!("runtime-reserved signal should be rejected");
19758        };
19759        assert!(message.contains("reserved by the workflow runtime"));
19760        assert!(ctx.take_commands().expect("commands").is_empty());
19761    }
19762
19763    #[tokio::test]
19764    async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19765        let client = Client::builder("http://127.0.0.1:9")
19766            .build()
19767            .expect("client");
19768        let error = client
19769            .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19770            .await
19771            .expect_err("runtime-reserved signal should be rejected before transport");
19772
19773        assert!(
19774            matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19775        );
19776    }
19777
19778    #[test]
19779    fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19780        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19781            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19782                value.to_string(),
19783            )]))
19784            .expect("message payload");
19785            json!({
19786                "schema": MESSAGE_STREAM_SCHEMA,
19787                "stream_name": "orders",
19788                "message_id": message_id,
19789                "position": position,
19790                "payload_envelope": payload,
19791            })
19792        }
19793
19794        fn opened(sequence: u64) -> HistoryEvent {
19795            history_event(
19796                "SignalWaitOpened",
19797                json!({
19798                    "sequence": sequence,
19799                    "signal_name": MESSAGE_STREAM_SIGNAL,
19800                }),
19801            )
19802        }
19803
19804        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19805            history_event(
19806                "SignalApplied",
19807                json!({
19808                    "sequence": sequence,
19809                    "signal_name": MESSAGE_STREAM_SIGNAL,
19810                    "value": fixture_envelope(json!([delivery])),
19811                }),
19812            )
19813        }
19814
19815        fn received(delivery: Value) -> HistoryEvent {
19816            history_event(
19817                "SignalReceived",
19818                json!({
19819                    "signal_name": MESSAGE_STREAM_SIGNAL,
19820                    "arguments": fixture_envelope(json!([delivery])),
19821                    "payload_codec": DEFAULT_CODEC,
19822                }),
19823            )
19824        }
19825
19826        let client = Client::new("http://127.0.0.1:8080").expect("client");
19827        let mut worker = Worker::new(client, "rust-workers");
19828        worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19829            let messages = ctx.message_stream("orders")?.receive(2).await?;
19830            Ok(json!(messages
19831                .into_iter()
19832                .map(|message| message.message_id)
19833                .collect::<Vec<_>>()))
19834        });
19835
19836        let first = delivery("message-1", 1, "one");
19837        let second = delivery("message-2", 2, "two");
19838        let batch = worker
19839            .execute_workflow_task_decision(workflow_task(
19840                "rust.message-stream-batch",
19841                vec![
19842                    opened(1),
19843                    received(first.clone()),
19844                    applied(1, first.clone()),
19845                    received(first.clone()),
19846                    received(second),
19847                ],
19848                DEFAULT_CODEC,
19849            ))
19850            .expect("worker task consumes the available batch");
19851
19852        assert_eq!(batch.commands.len(), 1);
19853        assert_eq!(batch.commands[0]["type"], "complete_workflow");
19854        assert_eq!(
19855            decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19856                .expect("workflow result"),
19857            json!(["message-1", "message-2"])
19858        );
19859        assert_eq!(
19860            batch.message_stream_cursors,
19861            vec![json!({"stream_name": "orders", "through_position": 2})]
19862        );
19863        assert!(batch.message_stream_waits.is_empty());
19864
19865        let partial = worker
19866            .execute_workflow_task_decision(workflow_task(
19867                "rust.message-stream-batch",
19868                vec![opened(1), received(first.clone()), applied(1, first)],
19869                DEFAULT_CODEC,
19870            ))
19871            .expect("worker task returns without waiting for a missing second item");
19872        assert_eq!(partial.commands.len(), 1);
19873        assert_eq!(partial.commands[0]["type"], "complete_workflow");
19874        assert_eq!(
19875            decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19876                .expect("workflow result"),
19877            json!(["message-1"])
19878        );
19879        assert_eq!(
19880            partial.message_stream_cursors,
19881            vec![json!({"stream_name": "orders", "through_position": 1})]
19882        );
19883        assert!(partial.message_stream_waits.is_empty());
19884    }
19885
19886    #[test]
19887    fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19888        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19889            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19890                value.to_string(),
19891            )]))
19892            .expect("message payload");
19893            json!({
19894                "schema": MESSAGE_STREAM_SCHEMA,
19895                "stream_name": "orders",
19896                "message_id": message_id,
19897                "position": position,
19898                "payload_envelope": payload,
19899            })
19900        }
19901
19902        fn opened(sequence: u64) -> HistoryEvent {
19903            history_event(
19904                "SignalWaitOpened",
19905                json!({
19906                    "sequence": sequence,
19907                    "signal_name": MESSAGE_STREAM_SIGNAL,
19908                }),
19909            )
19910        }
19911
19912        fn received(delivery: Value) -> HistoryEvent {
19913            history_event(
19914                "SignalReceived",
19915                json!({
19916                    "signal_name": MESSAGE_STREAM_SIGNAL,
19917                    "arguments": fixture_envelope(json!([delivery])),
19918                    "payload_codec": DEFAULT_CODEC,
19919                }),
19920            )
19921        }
19922
19923        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19924            history_event(
19925                "SignalApplied",
19926                json!({
19927                    "sequence": sequence,
19928                    "signal_name": MESSAGE_STREAM_SIGNAL,
19929                    "value": fixture_envelope(json!([delivery])),
19930                }),
19931            )
19932        }
19933
19934        let client = Client::new("http://127.0.0.1:8080").expect("client");
19935        let mut worker = Worker::new(client, "rust-workers");
19936        worker.register_workflow(
19937            "rust.message-stream-partial-batches",
19938            |ctx, _input| async move {
19939                let stream = ctx.message_stream("orders")?;
19940                let first = stream.receive(10).await?;
19941                let second = stream.receive(10).await?;
19942                Ok(json!([
19943                    first
19944                        .into_iter()
19945                        .map(|message| message.message_id)
19946                        .collect::<Vec<_>>(),
19947                    second
19948                        .into_iter()
19949                        .map(|message| message.message_id)
19950                        .collect::<Vec<_>>(),
19951                ]))
19952            },
19953        );
19954
19955        let first = delivery("message-1", 1, "one");
19956        let second = delivery("message-2", 2, "two");
19957        let decision = worker
19958            .execute_workflow_task_decision(workflow_task(
19959                "rust.message-stream-partial-batches",
19960                vec![
19961                    opened(1),
19962                    received(first.clone()),
19963                    applied(1, first),
19964                    opened(2),
19965                    received(second.clone()),
19966                    applied(2, second),
19967                ],
19968                DEFAULT_CODEC,
19969            ))
19970            .expect("cold replay preserves both authored receive boundaries");
19971
19972        assert_eq!(decision.commands.len(), 1);
19973        assert_eq!(decision.commands[0]["type"], "complete_workflow");
19974        assert_eq!(
19975            decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19976                .expect("workflow result"),
19977            json!([["message-1"], ["message-2"]])
19978        );
19979        assert_eq!(
19980            decision.message_stream_cursors,
19981            vec![json!({"stream_name": "orders", "through_position": 2})]
19982        );
19983        assert!(decision.message_stream_waits.is_empty());
19984    }
19985
19986    #[test]
19987    fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19988        let ctx = workflow_context(Vec::new());
19989        let stream = ctx.message_stream("orders").expect("message stream");
19990        let mut receive = Box::pin(stream.receive(10));
19991        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19992
19993        assert!(matches!(
19994            receive.as_mut().poll(&mut task_context),
19995            Poll::Pending
19996        ));
19997        assert_eq!(
19998            ctx.take_commands().expect("message-stream wait command"),
19999            vec![json!({
20000                "type": "open_signal_wait",
20001                "signal_name": MESSAGE_STREAM_SIGNAL,
20002            })]
20003        );
20004        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20005        assert!(cursors.is_empty());
20006        assert_eq!(
20007            waits,
20008            vec![json!({"stream_name": "orders", "after_position": 0})]
20009        );
20010    }
20011
20012    #[test]
20013    fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
20014        let ctx = workflow_context(vec![history_event(
20015            "SignalReceived",
20016            json!({
20017                "signal_name": MESSAGE_STREAM_SIGNAL,
20018                "arguments": fixture_envelope(json!([{
20019                    "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
20020                    "stream_name": "orders",
20021                    "through_position": 2,
20022                }])),
20023                "payload_codec": DEFAULT_CODEC,
20024            }),
20025        )]);
20026        let stream = ctx.message_stream("orders").expect("message stream");
20027        let mut receive = Box::pin(stream.receive(10));
20028        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20029
20030        assert!(matches!(
20031            receive.as_mut().poll(&mut task_context),
20032            Poll::Pending
20033        ));
20034        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20035        assert_eq!(
20036            cursors,
20037            vec![json!({"stream_name": "orders", "through_position": 2})]
20038        );
20039        assert_eq!(
20040            waits,
20041            vec![json!({"stream_name": "orders", "after_position": 2})]
20042        );
20043    }
20044
20045    #[test]
20046    fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
20047        let mut empty_map = BTreeMap::new();
20048        let mut nested = BTreeMap::new();
20049        nested.insert(
20050            "value".to_string(),
20051            AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
20052        );
20053        let values = vec![
20054            AvroValue::Bytes(vec![0, 255]),
20055            AvroValue::Long(1),
20056            AvroValue::Double(1.0),
20057            AvroValue::Array(Vec::new()),
20058            AvroValue::Map(std::mem::take(&mut empty_map)),
20059            AvroValue::Map(nested),
20060        ];
20061        let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
20062        let transport = vec![json!({
20063            "schema": MESSAGE_STREAM_SCHEMA,
20064            "stream_name": "orders",
20065            "message_id": "message-1",
20066            "position": 1,
20067            "payload_envelope": payload,
20068        })];
20069
20070        for _ in 0..2 {
20071            let Some(MessageStreamDelivery::Message(message)) =
20072                decode_message_stream_delivery(transport.clone()).expect("delivery")
20073            else {
20074                panic!("message delivery expected");
20075            };
20076            assert_eq!(message.arguments, values);
20077            assert!(matches!(message.arguments[1], AvroValue::Long(1)));
20078            assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
20079        }
20080    }
20081
20082    #[test]
20083    fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
20084        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20085            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20086                value.to_string(),
20087            )]))
20088            .expect("message payload");
20089            json!({
20090                "schema": MESSAGE_STREAM_SCHEMA,
20091                "stream_name": "orders",
20092                "message_id": message_id,
20093                "position": position,
20094                "payload_envelope": payload,
20095            })
20096        }
20097
20098        fn opened(sequence: u64) -> HistoryEvent {
20099            history_event(
20100                "SignalWaitOpened",
20101                json!({
20102                    "sequence": sequence,
20103                    "signal_name": MESSAGE_STREAM_SIGNAL,
20104                }),
20105            )
20106        }
20107
20108        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20109            history_event(
20110                "SignalApplied",
20111                json!({
20112                    "sequence": sequence,
20113                    "signal_name": MESSAGE_STREAM_SIGNAL,
20114                    "value": fixture_envelope(json!([delivery])),
20115                }),
20116            )
20117        }
20118
20119        fn worker() -> Worker {
20120            let client = Client::new("http://127.0.0.1:8080").expect("client");
20121            let mut worker = Worker::new(client, "rust-workers");
20122            worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20123                let stream = ctx.message_stream("orders")?;
20124                let first = stream.receive_one().await?;
20125                let second = stream.receive_one().await?;
20126                Ok(json!([first.message_id, second.message_id]))
20127            });
20128            worker
20129        }
20130
20131        fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20132            let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20133            task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20134            task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20135            task
20136        }
20137
20138        let waiting = worker()
20139            .execute_workflow_task_decision(workflow_task(
20140                "rust.message-stream",
20141                Vec::new(),
20142                DEFAULT_CODEC,
20143            ))
20144            .expect("first worker opens the stream wait");
20145        assert_eq!(
20146            waiting.commands,
20147            vec![json!({
20148                "type": "open_signal_wait",
20149                "signal_name": MESSAGE_STREAM_SIGNAL,
20150            })]
20151        );
20152        assert!(waiting.message_stream_cursors.is_empty());
20153        assert_eq!(
20154            waiting.message_stream_waits,
20155            vec![json!({"stream_name": "orders", "after_position": 0})]
20156        );
20157
20158        let first_delivery = delivery("message-1", 1, "one");
20159        let first_arrival = worker()
20160            .execute_workflow_task_decision(task_with_resume(
20161                vec![opened(1)],
20162                first_delivery.clone(),
20163            ))
20164            .expect("replacement worker consumes the first arrival");
20165        assert_eq!(
20166            first_arrival.commands,
20167            vec![json!({
20168                "type": "open_signal_wait",
20169                "signal_name": MESSAGE_STREAM_SIGNAL,
20170            })]
20171        );
20172        assert_eq!(
20173            first_arrival.message_stream_cursors,
20174            vec![json!({"stream_name": "orders", "through_position": 1})]
20175        );
20176        assert_eq!(
20177            first_arrival.message_stream_waits,
20178            vec![json!({"stream_name": "orders", "after_position": 1})]
20179        );
20180
20181        let second_delivery = delivery("message-2", 2, "two");
20182        let first_applied = applied(1, first_delivery);
20183        let completed = worker()
20184            .execute_workflow_task_decision(task_with_resume(
20185                vec![opened(1), first_applied.clone(), opened(2)],
20186                second_delivery.clone(),
20187            ))
20188            .expect("next replacement worker consumes the second arrival");
20189        assert_eq!(completed.commands.len(), 1);
20190        assert_eq!(completed.commands[0]["type"], "complete_workflow");
20191        assert_eq!(
20192            decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20193                .expect("workflow result"),
20194            json!(["message-1", "message-2"])
20195        );
20196        assert_eq!(
20197            completed.message_stream_cursors,
20198            vec![json!({"stream_name": "orders", "through_position": 2})]
20199        );
20200        assert!(completed.message_stream_waits.is_empty());
20201
20202        let replay_history = vec![
20203            opened(1),
20204            first_applied,
20205            opened(2),
20206            applied(2, second_delivery),
20207        ];
20208        for _cold_worker_or_restart in 0..2 {
20209            let replayed = worker()
20210                .execute_workflow_task_decision(workflow_task(
20211                    "rust.message-stream",
20212                    replay_history.clone(),
20213                    DEFAULT_CODEC,
20214                ))
20215                .expect("cold worker replays each logical message exactly once");
20216            assert_eq!(replayed.commands.len(), 1);
20217            assert_eq!(
20218                decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20219                    .expect("replayed workflow result"),
20220                json!(["message-1", "message-2"])
20221            );
20222            assert_eq!(
20223                replayed.message_stream_cursors,
20224                vec![json!({"stream_name": "orders", "through_position": 2})]
20225            );
20226            assert!(replayed.message_stream_waits.is_empty());
20227        }
20228    }
20229
20230    #[test]
20231    fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20232        assert!(!worker_protocol_supports_message_streams("1.14"));
20233        assert!(worker_protocol_supports_message_streams("1.15"));
20234        assert!(worker_protocol_supports_message_streams("1.16"));
20235        assert!(worker_protocol_supports_message_streams(
20236            WORKER_PROTOCOL_VERSION
20237        ));
20238        assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20239    }
20240
20241    #[test]
20242    fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20243        let ctx = workflow_context(vec![
20244            history_event(
20245                "ConditionWaitOpened",
20246                json!({
20247                    "sequence": 1,
20248                    "condition_wait_id": "condition:1",
20249                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20250                    "condition_key": "signal:finish",
20251                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20252                }),
20253            ),
20254            history_event(
20255                "ConditionWaitSatisfied",
20256                json!({
20257                    "sequence": 1,
20258                    "condition_wait_id": "condition:1",
20259                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20260                    "condition_key": "signal:finish",
20261                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20262                }),
20263            ),
20264            history_event(
20265                "SignalReceived",
20266                json!({"signal_name": "finish", "arguments": []}),
20267            ),
20268        ]);
20269        let mut signal = Box::pin(ctx.wait_signal("finish"));
20270        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20271
20272        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20273            signal.as_mut().poll(&mut task_context)
20274        else {
20275            panic!("condition history must not resolve as a typed signal wait");
20276        };
20277        assert_eq!(failure.reason, "recorded_command_mismatch");
20278        assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20279    }
20280
20281    #[test]
20282    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20283        let signal_then_timer = vec![
20284            history_event(
20285                "SignalWaitOpened",
20286                json!({"sequence": 1, "signal_name": "go"}),
20287            ),
20288            history_event(
20289                "SignalApplied",
20290                json!({
20291                    "sequence": 1,
20292                    "signal_name": "go",
20293                    "value": fixture_envelope(json!(["now"])),
20294                }),
20295            ),
20296            history_event(
20297                "TimerScheduled",
20298                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20299            ),
20300            history_event(
20301                "TimerFired",
20302                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20303            ),
20304        ];
20305
20306        let ctx = workflow_context(signal_then_timer.clone());
20307        let mut signal = Box::pin(ctx.wait_signal("go"));
20308        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20309        assert!(matches!(
20310            signal.as_mut().poll(&mut task_context),
20311            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20312        ));
20313        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20314        assert!(matches!(
20315            timer.as_mut().poll(&mut task_context),
20316            Poll::Ready(Ok(()))
20317        ));
20318        ctx.ensure_history_consumed()
20319            .expect("signal and timer history consumed in order");
20320
20321        let reordered = workflow_context(signal_then_timer);
20322        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20323        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20324            timer_first.as_mut().poll(&mut task_context)
20325        else {
20326            panic!("timer cannot consume signal-wait-first history");
20327        };
20328        assert_eq!(failure.reason, "recorded_command_mismatch");
20329        assert_eq!(failure.sequence, Some(1));
20330        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20331
20332        let timer_then_signal = vec![
20333            history_event(
20334                "TimerScheduled",
20335                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20336            ),
20337            history_event(
20338                "TimerFired",
20339                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20340            ),
20341            history_event(
20342                "SignalWaitOpened",
20343                json!({"sequence": 2, "signal_name": "go"}),
20344            ),
20345            history_event(
20346                "SignalApplied",
20347                json!({
20348                    "sequence": 2,
20349                    "signal_name": "go",
20350                    "value": fixture_envelope(json!([])),
20351                }),
20352            ),
20353        ];
20354        let reordered = workflow_context(timer_then_signal);
20355        let mut signal_first = Box::pin(reordered.wait_signal("go"));
20356        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20357            signal_first.as_mut().poll(&mut task_context)
20358        else {
20359            panic!("signal wait cannot consume timer-first history");
20360        };
20361        assert_eq!(failure.reason, "recorded_command_mismatch");
20362        assert_eq!(failure.sequence, Some(1));
20363        assert_eq!(failure.expected.as_deref(), Some("timer"));
20364    }
20365
20366    #[test]
20367    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20368        let duplicate_timer = WorkflowState::new(
20369            vec![
20370                history_event(
20371                    "TimerScheduled",
20372                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20373                ),
20374                history_event(
20375                    "TimerScheduled",
20376                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20377                ),
20378            ],
20379            "rust-workers".to_string(),
20380            DEFAULT_CODEC.to_string(),
20381            None,
20382        )
20383        .expect_err("one workflow sequence cannot schedule two timers");
20384        assert!(matches!(
20385            duplicate_timer,
20386            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20387                if reason == "timer_schedule_missing_or_duplicate"
20388        ));
20389
20390        let colliding_kinds = WorkflowState::new(
20391            vec![
20392                history_event(
20393                    "TimerScheduled",
20394                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20395                ),
20396                history_event(
20397                    "ActivityCompleted",
20398                    json!({"sequence": 1, "activity_type": "same-sequence"}),
20399                ),
20400            ],
20401            "rust-workers".to_string(),
20402            DEFAULT_CODEC.to_string(),
20403            None,
20404        )
20405        .expect_err("one workflow sequence cannot identify two command kinds");
20406        assert!(matches!(
20407            colliding_kinds,
20408            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20409                if reason == "durable_command_sequence_collision"
20410        ));
20411
20412        let duplicate_signal_wait = WorkflowState::new(
20413            vec![
20414                history_event(
20415                    "SignalWaitOpened",
20416                    json!({"sequence": 1, "signal_name": "go"}),
20417                ),
20418                history_event(
20419                    "SignalWaitOpened",
20420                    json!({"sequence": 1, "signal_name": "go"}),
20421                ),
20422            ],
20423            "rust-workers".to_string(),
20424            DEFAULT_CODEC.to_string(),
20425            None,
20426        )
20427        .expect_err("one workflow sequence cannot open two signal waits");
20428        assert!(matches!(
20429            duplicate_signal_wait,
20430            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20431                if reason == "signal_wait_open_missing_or_duplicate"
20432        ));
20433    }
20434
20435    #[test]
20436    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20437        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20438            .expect("side-effect result");
20439        let ctx = workflow_context(vec![history_event(
20440            "SideEffectRecorded",
20441            json!({"sequence": 99, "result": result}),
20442        )]);
20443
20444        let replayed: Value = ctx
20445            .side_effect(|| panic!("recorded side effect must not run"))
20446            .expect("positive global workflow sequence is valid");
20447        assert_eq!(replayed, json!({"captured": true}));
20448        ctx.ensure_history_consumed().expect("history consumed");
20449    }
20450
20451    #[test]
20452    fn workflow_history_rejects_zero_and_descending_command_sequences() {
20453        let result =
20454            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20455        let zero = WorkflowState::new(
20456            vec![history_event(
20457                "SideEffectRecorded",
20458                json!({"sequence": 0, "result": result.clone()}),
20459            )],
20460            "rust-workers".to_string(),
20461            DEFAULT_CODEC.to_string(),
20462            None,
20463        )
20464        .expect_err("durable command sequences must be positive");
20465        assert!(matches!(
20466            zero,
20467            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20468                if reason == "durable_command_sequence_invalid"
20469        ));
20470
20471        let descending = WorkflowState::new(
20472            vec![
20473                history_event(
20474                    "SideEffectRecorded",
20475                    json!({"sequence": 3, "result": result}),
20476                ),
20477                history_event(
20478                    "VersionMarkerRecorded",
20479                    json!({
20480                        "sequence": 2,
20481                        "change_id": "descending-marker",
20482                        "version": 1,
20483                        "min_supported": 1,
20484                        "max_supported": 1,
20485                    }),
20486                ),
20487            ],
20488            "rust-workers".to_string(),
20489            DEFAULT_CODEC.to_string(),
20490            None,
20491        )
20492        .expect_err("new durable commands must remain strictly ordered");
20493        let Error::NonDeterministicReplay(failure) = descending else {
20494            panic!("expected typed replay failure");
20495        };
20496        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20497        assert_eq!(failure.sequence, Some(2));
20498        assert_eq!(
20499            failure.expected.as_deref(),
20500            Some("workflow sequence greater than 3")
20501        );
20502        assert_eq!(failure.actual.as_deref(), Some("2"));
20503    }
20504
20505    #[test]
20506    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20507        fn worker() -> Worker {
20508            let client = Client::new("http://127.0.0.1:8080").expect("client");
20509            let mut worker = Worker::new(client, "rust-workers");
20510            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20511                ctx.wait_signal("finish").await?;
20512                let marker: String =
20513                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20514                assert_eq!(marker, "after-finish");
20515                Ok(json!("finished"))
20516            });
20517            worker
20518        }
20519
20520        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20521            .expect("side-effect result");
20522        let task = workflow_task(
20523            "rust.finish-after-gaps",
20524            vec![
20525                history_event(
20526                    "SignalWaitOpened",
20527                    json!({"sequence": 1, "signal_name": "finish"}),
20528                ),
20529                history_event(
20530                    "SignalReceived",
20531                    json!({
20532                        "signal_id": "increment-3",
20533                        "signal_name": "increment",
20534                        "workflow_sequence": 2,
20535                        "payload_codec": DEFAULT_CODEC,
20536                        "arguments": fixture_envelope(json!([3])),
20537                    }),
20538                ),
20539                history_event(
20540                    "SignalReceived",
20541                    json!({
20542                        "signal_id": "increment-5",
20543                        "signal_name": "increment",
20544                        "workflow_sequence": 3,
20545                        "payload_codec": DEFAULT_CODEC,
20546                        "arguments": fixture_envelope(json!([5])),
20547                    }),
20548                ),
20549                history_event(
20550                    "SignalReceived",
20551                    json!({
20552                        "signal_id": "finish",
20553                        "signal_name": "finish",
20554                        "workflow_sequence": 4,
20555                        "payload_codec": DEFAULT_CODEC,
20556                        "arguments": fixture_envelope(json!([])),
20557                    }),
20558                ),
20559                history_event(
20560                    "SignalApplied",
20561                    json!({
20562                        "sequence": 1,
20563                        "signal_id": "finish",
20564                        "signal_name": "finish",
20565                        "payload_codec": DEFAULT_CODEC,
20566                        "value": fixture_envelope(json!([])),
20567                    }),
20568                ),
20569                history_event(
20570                    "SideEffectRecorded",
20571                    json!({"sequence": 5, "result": marker}),
20572                ),
20573            ],
20574            DEFAULT_CODEC,
20575        );
20576
20577        for _original_or_cold_worker in 0..2 {
20578            let commands = worker()
20579                .execute_workflow_task(task.clone())
20580                .expect("signal gaps preserve deterministic replay");
20581            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20582            assert_eq!(commands[0]["type"], "complete_workflow");
20583            assert_eq!(
20584                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20585                json!("finished")
20586            );
20587        }
20588    }
20589
20590    #[test]
20591    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20592        let ctx = workflow_context(Vec::new());
20593        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20594        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20595        assert!(matches!(
20596            sleep.as_mut().poll(&mut task_context),
20597            Poll::Ready(Err(Error::TimerDurationOverflow))
20598        ));
20599        assert!(ctx.take_commands().expect("commands").is_empty());
20600    }
20601
20602    #[test]
20603    fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20604        let entries = AvroValue::Map(BTreeMap::from([
20605            ("text".to_string(), AvroValue::String("same".to_string())),
20606            (
20607                "nested".to_string(),
20608                AvroValue::Map(BTreeMap::from([
20609                    ("beta".to_string(), AvroValue::Long(2)),
20610                    ("alpha".to_string(), AvroValue::Long(1)),
20611                ])),
20612            ),
20613            ("long".to_string(), AvroValue::Long(7)),
20614            ("double".to_string(), AvroValue::Double(7.0)),
20615            ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20616        ]));
20617        let ctx = workflow_context(Vec::new());
20618        ctx.upsert_memo(entries.clone()).expect("valid memo update");
20619        let commands = ctx.take_commands().expect("commands");
20620
20621        assert_eq!(commands.len(), 1);
20622        assert_eq!(commands[0]["type"], "upsert_memo");
20623        let server_entries = json!({
20624            "codec": "avro",
20625            "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20626        });
20627        assert_eq!(
20628            commands[0]["entries"]
20629                .as_object()
20630                .expect("entries envelope")
20631                .keys()
20632                .collect::<Vec<_>>(),
20633            vec!["blob", "codec"]
20634        );
20635        assert_eq!(commands[0]["entries"], server_entries);
20636        let wire_entries =
20637            decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20638        assert_eq!(wire_entries, entries);
20639
20640        let history = vec![history_event(
20641            "MemoUpserted",
20642            json!({
20643                "sequence": 1,
20644                "entries": server_entries.clone(),
20645                "merged": server_entries,
20646            }),
20647        )];
20648        let replay = workflow_context(history.clone());
20649        replay
20650            .upsert_memo(entries.clone())
20651            .expect("matching replay identity");
20652        assert!(replay.take_commands().expect("replay commands").is_empty());
20653
20654        let changed_types = AvroValue::Map(BTreeMap::from([
20655            ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20656            (
20657                "nested".to_string(),
20658                AvroValue::Map(BTreeMap::from([
20659                    ("alpha".to_string(), AvroValue::Long(1)),
20660                    ("beta".to_string(), AvroValue::Long(2)),
20661                ])),
20662            ),
20663            ("long".to_string(), AvroValue::Double(7.0)),
20664            ("double".to_string(), AvroValue::Long(7)),
20665            ("binary".to_string(), AvroValue::String("same".to_string())),
20666        ]));
20667        let error = workflow_context(history)
20668            .upsert_memo(changed_types)
20669            .expect_err("memo replay identity must preserve Avro value types");
20670        assert!(matches!(
20671            error,
20672            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20673        ));
20674    }
20675
20676    #[test]
20677    fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20678        let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20679            .expect("memo envelope");
20680        let replay = workflow_context(vec![history_event(
20681            "MemoUpserted",
20682            json!({
20683                "sequence": 1,
20684                "entries": original.clone(),
20685                "merged": original
20686            }),
20687        )]);
20688        let error = replay
20689            .upsert_memo(json!({"stage": "changed"}))
20690            .expect_err("changed memo update must fail replay");
20691        assert!(matches!(
20692            error,
20693            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20694        ));
20695
20696        let invalid = workflow_context(Vec::new())
20697            .upsert_memo(
20698                json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20699            )
20700            .expect_err("oversized key");
20701        assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20702    }
20703
20704    #[test]
20705    fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20706        let negative_zero = AvroValue::Map(BTreeMap::from([(
20707            "reading".to_string(),
20708            AvroValue::Double(-0.0),
20709        )]));
20710        let negative_zero_envelope =
20711            encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20712        let history = vec![history_event(
20713            "MemoUpserted",
20714            json!({
20715                "sequence": 1,
20716                "entries": negative_zero_envelope.clone(),
20717                "merged": negative_zero_envelope,
20718            }),
20719        )];
20720
20721        workflow_context(history.clone())
20722            .upsert_memo(negative_zero)
20723            .expect("matching negative-zero history identity");
20724
20725        let error = workflow_context(history)
20726            .upsert_memo(AvroValue::Map(BTreeMap::from([(
20727                "reading".to_string(),
20728                AvroValue::Double(0.0),
20729            )])))
20730            .expect_err("positive zero must not consume negative-zero memo history");
20731        assert!(matches!(
20732            error,
20733            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20734        ));
20735    }
20736
20737    #[test]
20738    fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20739        let supported = json!({
20740            "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20741            "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20742        });
20743        assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20744        assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20745            "workflow_memo_updates": {"supported": false},
20746            "supported_workflow_task_commands": ["upsert_memo"]
20747        }))));
20748        assert!(commands_use_workflow_memo_updates(&[json!({
20749            "type": "upsert_memo",
20750            "entries": {"stage": "processing"}
20751        })]));
20752    }
20753
20754    #[test]
20755    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20756        let client = Client::new("http://127.0.0.1:8080").expect("client");
20757        let mut worker = Worker::new(client, "rust-workers");
20758        worker.register_workflow("rust.timer", |ctx, _input| async move {
20759            ctx.sleep(Duration::from_secs(5)).await?;
20760            ctx.activity("after-timer", json!([])).await
20761        });
20762
20763        let task = |history_events| WorkflowTask {
20764            task_id: "wft-rust-timer-1".to_string(),
20765            workflow_command_id: None,
20766            workflow_id: Some("wf-rust-timer".to_string()),
20767            run_id: Some("run-rust-timer".to_string()),
20768            workflow_type: "rust.timer".to_string(),
20769            cancel_requested: false,
20770            payload_codec: DEFAULT_CODEC.to_string(),
20771            arguments: Some(
20772                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20773            ),
20774            history_events,
20775            total_history_events: None,
20776            history_size_bytes: None,
20777            continue_as_new_recommended: None,
20778            history_budget_pressure: None,
20779            next_history_page_token: None,
20780            workflow_task_attempt: 1,
20781            workflow_signal_id: None,
20782            signal_name: None,
20783            signal_arguments: None,
20784            workflow_update_id: None,
20785            update_name: None,
20786            lease_owner: Some("rust-worker".to_string()),
20787        };
20788
20789        let initial = worker
20790            .execute_workflow_task(task(Vec::new()))
20791            .expect("initial timer task");
20792        assert_eq!(
20793            initial,
20794            vec![json!({"type": "start_timer", "delay_seconds": 5})]
20795        );
20796
20797        let activity_result =
20798            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20799        let replayed = worker
20800            .execute_workflow_task(task(vec![
20801                history_event(
20802                    "TimerScheduled",
20803                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20804                ),
20805                history_event(
20806                    "TimerFired",
20807                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20808                ),
20809                history_event(
20810                    "ActivityCompleted",
20811                    json!({
20812                        "sequence": 2,
20813                        "activity_type": "after-timer",
20814                        "payload_codec": DEFAULT_CODEC,
20815                        "result": activity_result,
20816                    }),
20817                ),
20818            ]))
20819            .expect("replayed workflow task");
20820        assert_eq!(replayed.len(), 1);
20821        assert_eq!(replayed[0]["type"], "complete_workflow");
20822        assert_eq!(
20823            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20824            json!("done")
20825        );
20826    }
20827
20828    #[test]
20829    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20830        let client = Client::new("http://127.0.0.1:8080").expect("client");
20831        let mut worker = Worker::new(client, "rust-workers");
20832        worker.register_workflow("rust.continue", |ctx, _input| async move {
20833            ctx.continue_as_new_with_options(
20834                ContinueAsNewOptions::new()
20835                    .workflow_type("rust.next")
20836                    .task_queue("next-workers"),
20837                json!([2, {"cursor": "next"}]),
20838            )
20839        });
20840
20841        let commands = worker
20842            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20843            .expect("continue-as-new command");
20844
20845        assert_eq!(commands.len(), 1);
20846        assert_eq!(commands[0]["type"], "continue_as_new");
20847        assert_eq!(commands[0]["workflow_type"], "rust.next");
20848        assert_eq!(commands[0]["queue"], "next-workers");
20849        assert_eq!(
20850            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20851                .expect("continue-as-new arguments"),
20852            json!([2, {"cursor": "next"}])
20853        );
20854    }
20855
20856    #[test]
20857    fn continue_as_new_preserves_typed_arguments() {
20858        let client = Client::new("http://127.0.0.1:8080").expect("client");
20859        let mut worker = Worker::new(client, "rust-workers");
20860        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20861            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20862            unreachable!("continue-as-new returns a control-flow error")
20863        });
20864
20865        let commands = worker
20866            .execute_workflow_task(workflow_task(
20867                "rust.typed-continue",
20868                Vec::new(),
20869                DEFAULT_CODEC,
20870            ))
20871            .expect("typed continue-as-new command");
20872
20873        assert_eq!(commands[0]["type"], "continue_as_new");
20874        assert_eq!(
20875            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20876                .expect("typed continue arguments"),
20877            AvroValue::Array(vec![typed_fidelity_probe()])
20878        );
20879    }
20880
20881    #[test]
20882    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20883        let client = Client::new("http://127.0.0.1:8080").expect("client");
20884        let mut worker = Worker::new(client, "rust-workers");
20885        worker.register_workflow("rust.continue", |ctx, _input| async move {
20886            ctx.continue_as_new(json!([2]))
20887        });
20888        let task = workflow_task(
20889            "rust.continue",
20890            vec![history_event(
20891                "WorkflowContinuedAsNew",
20892                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20893            )],
20894            DEFAULT_CODEC,
20895        );
20896
20897        for _worker_restart_or_redelivery in 0..2 {
20898            let commands = worker
20899                .execute_workflow_task(task.clone())
20900                .expect("recorded transition replays");
20901            assert!(
20902                commands.is_empty(),
20903                "replay must not emit another successor"
20904            );
20905        }
20906    }
20907
20908    #[test]
20909    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20910        let ctx = workflow_context(Vec::new());
20911        let error = ctx
20912            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
20913            .expect_err("blank queue must be rejected");
20914
20915        let Error::InvalidContinueAsNewOptions(error) = error else {
20916            panic!("expected typed continue-as-new validation error");
20917        };
20918        assert_eq!(error.field, "task_queue");
20919        assert!(ctx.take_commands().expect("commands").is_empty());
20920    }
20921
20922    #[test]
20923    fn workflow_context_exposes_server_history_budget() {
20924        let client = Client::new("http://127.0.0.1:8080").expect("client");
20925        let mut worker = Worker::new(client, "rust-workers");
20926        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20927            let budget = ctx.history_budget()?;
20928            Ok(json!({
20929                "events": budget.event_count,
20930                "bytes": budget.size_bytes,
20931                "recommended": budget.continue_as_new_recommended,
20932                "pressure": budget.pressure,
20933            }))
20934        });
20935        let task: WorkflowTask = serde_json::from_value(json!({
20936            "task_id": "task-history-budget",
20937            "workflow_type": "rust.history-budget",
20938            "payload_codec": DEFAULT_CODEC,
20939            "history_events": [],
20940            "total_history_events": 480,
20941            "history_size_bytes": 1_048_576,
20942            "continue_as_new_recommended": true,
20943            "history_budget_pressure": "continue_as_new_recommended",
20944        }))
20945        .expect("published workflow task");
20946
20947        let commands = worker
20948            .execute_workflow_task(task)
20949            .expect("history-budget workflow");
20950        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20951        assert_eq!(result["events"], 480);
20952        assert_eq!(result["bytes"], 1_048_576);
20953        assert_eq!(result["recommended"], true);
20954        assert_eq!(result["pressure"], "continue_as_new_recommended");
20955    }
20956
20957    #[test]
20958    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20959        let client = Client::new("http://127.0.0.1:8080").expect("client");
20960        let mut worker = Worker::new(client, "rust-workers");
20961        worker.register_workflow("rust.failing", |_ctx, _input| async move {
20962            Err(Error::Codec("rust_conformance_failure".to_string()))
20963        });
20964        let task = WorkflowTask {
20965            task_id: "wft-rust-failing-1".to_string(),
20966            workflow_command_id: None,
20967            workflow_id: Some("wf-rust-failing".to_string()),
20968            run_id: Some("run-rust-failing".to_string()),
20969            workflow_type: "rust.failing".to_string(),
20970            cancel_requested: false,
20971            payload_codec: DEFAULT_CODEC.to_string(),
20972            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20973            history_events: Vec::new(),
20974            total_history_events: Some(0),
20975            history_size_bytes: None,
20976            continue_as_new_recommended: None,
20977            history_budget_pressure: None,
20978            next_history_page_token: None,
20979            workflow_task_attempt: 1,
20980            workflow_signal_id: None,
20981            signal_name: None,
20982            signal_arguments: None,
20983            workflow_update_id: None,
20984            update_name: None,
20985            lease_owner: Some("rust-worker".to_string()),
20986        };
20987
20988        let commands = worker
20989            .execute_workflow_task(task)
20990            .expect("handler failure becomes a workflow command");
20991
20992        assert_eq!(commands.len(), 1);
20993        assert_eq!(commands[0]["type"], "fail_workflow");
20994        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20995        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20996        assert_eq!(commands[0]["non_retryable"], false);
20997        assert_eq!(
20998            commands[0]["message"],
20999            "codec error: rust_conformance_failure"
21000        );
21001        assert_eq!(
21002            commands[0]["exception"]["message"],
21003            "codec error: rust_conformance_failure"
21004        );
21005    }
21006
21007    #[test]
21008    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
21009        let client = Client::new("http://127.0.0.1:8080").expect("client");
21010        let mut worker = Worker::new(client, "rust-workers");
21011        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
21012            let _: String = ctx.side_effect(|| "captured".to_string())?;
21013            Err(Error::WorkerLoop("application failure".to_string()))
21014        });
21015
21016        let commands = worker
21017            .execute_workflow_task(workflow_task(
21018                "rust.failing-after-side-effect",
21019                Vec::new(),
21020                DEFAULT_CODEC,
21021            ))
21022            .expect("ordinary failure remains a workflow decision");
21023
21024        assert_eq!(commands.len(), 2);
21025        assert_eq!(commands[0]["type"], "record_side_effect");
21026        assert_eq!(commands[1]["type"], "fail_workflow");
21027    }
21028
21029    #[test]
21030    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
21031        let client = Client::new("http://127.0.0.1:8080").expect("client");
21032        let mut worker = Worker::new(client, "rust-workers");
21033        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
21034            Err(Error::WorkerLoop("application failure".to_string()))
21035        });
21036        let result =
21037            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
21038
21039        let error = worker
21040            .execute_workflow_task(workflow_task(
21041                "rust.removed-side-effect",
21042                vec![history_event(
21043                    "SideEffectRecorded",
21044                    json!({"sequence": 1, "result": result}),
21045                )],
21046                DEFAULT_CODEC,
21047            ))
21048            .expect_err("removed committed history must not become fail_workflow");
21049
21050        let Error::NonDeterministicReplay(failure) = error else {
21051            panic!("expected typed replay failure");
21052        };
21053        assert_eq!(failure.reason, "recorded_commands_unconsumed");
21054        assert_eq!(failure.sequence, Some(1));
21055        assert_eq!(failure.expected.as_deref(), Some("side effect"));
21056    }
21057
21058    #[test]
21059    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
21060        let client = Client::new("http://127.0.0.1:8080").expect("client");
21061        let mut worker = Worker::new(client, "rust-workers");
21062        worker.register_workflow(
21063            "rust.side-effect-before-marker-error",
21064            |ctx, _input| async move {
21065                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
21066                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
21067                ctx.get_version("restart-safe", 2, 2)?;
21068                Ok(Value::Null)
21069            },
21070        );
21071
21072        let error = worker
21073            .execute_workflow_task(workflow_task(
21074                "rust.side-effect-before-marker-error",
21075                vec![history_event(
21076                    "VersionMarkerRecorded",
21077                    json!({
21078                        "sequence": 1,
21079                        "change_id": "restart-safe",
21080                        "version": 1,
21081                        "min_supported": 1,
21082                        "max_supported": 1,
21083                    }),
21084                )],
21085                DEFAULT_CODEC,
21086            ))
21087            .expect_err("replay error must return no queued workflow commands");
21088
21089        let Error::NonDeterministicReplay(failure) = error else {
21090            panic!("expected typed replay failure");
21091        };
21092        assert_eq!(failure.reason, "version_marker_incompatible_range");
21093        assert_eq!(failure.sequence, Some(1));
21094    }
21095
21096    #[test]
21097    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
21098        let client = Client::new("http://127.0.0.1:8080").expect("client");
21099        let mut worker = Worker::new(client, "rust-workers");
21100        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
21101            ctx.sleep(Duration::from_secs(5)).await?;
21102            Ok(json!({"status": "timer fired"}))
21103        });
21104
21105        let task = WorkflowTask {
21106            task_id: "wft-rust-timer-pending".to_string(),
21107            workflow_command_id: None,
21108            workflow_id: Some("wf-rust-timer".to_string()),
21109            run_id: Some("run-rust-timer".to_string()),
21110            workflow_type: "rust.timer.pending".to_string(),
21111            cancel_requested: false,
21112            payload_codec: DEFAULT_CODEC.to_string(),
21113            arguments: Some(
21114                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21115            ),
21116            history_events: vec![history_event(
21117                "TimerScheduled",
21118                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21119            )],
21120            total_history_events: Some(1),
21121            history_size_bytes: None,
21122            continue_as_new_recommended: None,
21123            history_budget_pressure: None,
21124            next_history_page_token: None,
21125            workflow_task_attempt: 1,
21126            workflow_signal_id: None,
21127            signal_name: None,
21128            signal_arguments: None,
21129            workflow_update_id: None,
21130            update_name: None,
21131            lease_owner: Some("rust-worker".to_string()),
21132        };
21133
21134        for _redelivery_or_restart in 0..2 {
21135            let commands = worker
21136                .execute_workflow_task(task.clone())
21137                .expect("recorded timer remains pending");
21138            assert!(
21139                commands.is_empty(),
21140                "recorded timer must not be rescheduled"
21141            );
21142        }
21143    }
21144
21145    #[test]
21146    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21147        let client = Client::new("http://127.0.0.1:8080").expect("client");
21148        let mut worker = Worker::new(client, "rust-workers");
21149        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21150            Ok(json!({"status": "completed"}))
21151        });
21152        let task = WorkflowTask {
21153            task_id: "wft-rust-timer-removed".to_string(),
21154            workflow_command_id: None,
21155            workflow_id: Some("wf-rust-timer".to_string()),
21156            run_id: Some("run-rust-timer".to_string()),
21157            workflow_type: "rust.timer.removed".to_string(),
21158            cancel_requested: false,
21159            payload_codec: DEFAULT_CODEC.to_string(),
21160            arguments: Some(
21161                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21162            ),
21163            history_events: vec![
21164                history_event(
21165                    "TimerScheduled",
21166                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21167                ),
21168                history_event(
21169                    "TimerFired",
21170                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21171                ),
21172            ],
21173            total_history_events: Some(2),
21174            history_size_bytes: None,
21175            continue_as_new_recommended: None,
21176            history_budget_pressure: None,
21177            next_history_page_token: None,
21178            workflow_task_attempt: 1,
21179            workflow_signal_id: None,
21180            signal_name: None,
21181            signal_arguments: None,
21182            workflow_update_id: None,
21183            update_name: None,
21184            lease_owner: Some("rust-worker".to_string()),
21185        };
21186
21187        let Error::NonDeterministicReplay(failure) = worker
21188            .execute_workflow_task(task)
21189            .expect_err("removed timer must fail replay")
21190        else {
21191            panic!("expected typed replay failure");
21192        };
21193        assert_eq!(failure.reason, "recorded_commands_unconsumed");
21194        assert_eq!(failure.sequence, Some(1));
21195    }
21196
21197    #[test]
21198    fn workflow_context_emits_explicit_child_workflow_contract() {
21199        let ctx = WorkflowContext {
21200            state: Arc::new(Mutex::new(
21201                WorkflowState::new_with_identity(
21202                    Vec::new(),
21203                    Some("wf-parent".to_string()),
21204                    Some("run-parent".to_string()),
21205                    "parent-workers".to_string(),
21206                    DEFAULT_CODEC.to_string(),
21207                    None,
21208                )
21209                .expect("workflow state"),
21210            )),
21211        };
21212        let options = ChildWorkflowOptions::new("python-workers")
21213            .parent_close_policy(ParentClosePolicy::RequestCancel)
21214            .retry_policy(ChildWorkflowRetryPolicy {
21215                max_attempts: Some(3),
21216                backoff_seconds: vec![1, 5],
21217                non_retryable_error_types: vec!["ValidationError".to_string()],
21218            })
21219            .execution_timeout_seconds(600)
21220            .run_timeout_seconds(120);
21221        let mut call = Box::pin(ctx.start_child_workflow(
21222            "python.fulfil-order",
21223            options,
21224            json!([{"order_id": "order-42"}]),
21225        ));
21226        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21227
21228        assert!(matches!(
21229            call.as_mut().poll(&mut task_context),
21230            Poll::Pending
21231        ));
21232        let commands = ctx.take_commands().expect("commands");
21233        assert_eq!(commands.len(), 1);
21234        let command = &commands[0];
21235        assert_eq!(command["type"], "start_child_workflow");
21236        assert_eq!(command["workflow_type"], "python.fulfil-order");
21237        assert_eq!(command["queue"], "python-workers");
21238        assert_eq!(command["parent_close_policy"], "request_cancel");
21239        assert_eq!(command["retry_policy"]["max_attempts"], 3);
21240        assert_eq!(command["execution_timeout_seconds"], 600);
21241        assert_eq!(command["run_timeout_seconds"], 120);
21242        assert_eq!(
21243            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21244            json!([{"order_id": "order-42"}])
21245        );
21246    }
21247
21248    fn child_parent_worker() -> Worker {
21249        let client = Client::new("http://127.0.0.1:8080").expect("client");
21250        let mut worker = Worker::new(client, "rust-parent-workers");
21251        worker.register_workflow("rust.parent", |ctx, _input| async move {
21252            let child = ctx
21253                .start_child_workflow(
21254                    "python.child",
21255                    ChildWorkflowOptions::new("python-child-workers")
21256                        .parent_close_policy(ParentClosePolicy::Terminate),
21257                    json!([{"codec_probe": [1, true, "rust"]}]),
21258                )
21259                .await?;
21260            Ok(json!({
21261                "parent_workflow_id": child.parent.workflow_id,
21262                "parent_run_id": child.parent.run_id,
21263                "child_workflow_id": child.child.workflow_id,
21264                "child_run_id": child.child.run_id,
21265                "child_workflow_type": child.child_workflow_type,
21266                "result": child.result,
21267            }))
21268        });
21269        worker
21270    }
21271
21272    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21273        WorkflowTask {
21274            task_id: "wft-child-parent".to_string(),
21275            workflow_command_id: None,
21276            workflow_id: Some("wf-parent".to_string()),
21277            run_id: Some("run-parent".to_string()),
21278            workflow_type: "rust.parent".to_string(),
21279            cancel_requested: false,
21280            payload_codec: DEFAULT_CODEC.to_string(),
21281            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21282            history_events: vec![
21283                HistoryEvent {
21284                    event_type: "ChildWorkflowScheduled".to_string(),
21285                    payload: json!({
21286                        "sequence": 1,
21287                        "child_call_id": "call-child",
21288                        "child_workflow_instance_id": "wf-child",
21289                        "child_workflow_run_id": "run-child",
21290                        "child_workflow_type": "python.child",
21291                    }),
21292                    raw: HashMap::new(),
21293                },
21294                HistoryEvent {
21295                    event_type: event_type.to_string(),
21296                    payload,
21297                    raw: HashMap::new(),
21298                },
21299            ],
21300            total_history_events: Some(2),
21301            history_size_bytes: None,
21302            continue_as_new_recommended: None,
21303            history_budget_pressure: None,
21304            next_history_page_token: None,
21305            workflow_task_attempt: 1,
21306            workflow_signal_id: None,
21307            signal_name: None,
21308            signal_arguments: None,
21309            workflow_update_id: None,
21310            update_name: None,
21311            lease_owner: Some("rust-worker".to_string()),
21312        }
21313    }
21314
21315    #[test]
21316    fn committed_child_result_replays_without_starting_a_duplicate() {
21317        let worker = child_parent_worker();
21318        let task = child_parent_task(
21319            "ChildRunCompleted",
21320            json!({
21321                "sequence": 1,
21322                "child_call_id": "call-child",
21323                "child_workflow_instance_id": "wf-child",
21324                "child_workflow_run_id": "run-child",
21325                "child_workflow_type": "python.child",
21326                "payload_codec": DEFAULT_CODEC,
21327                "result": fixture_envelope(json!({"from":"python","ok":true})),
21328            }),
21329        );
21330
21331        for _restart in 0..2 {
21332            let commands = worker
21333                .execute_workflow_task(task.clone())
21334                .expect("replayed parent task");
21335            assert_eq!(commands.len(), 1);
21336            assert_eq!(commands[0]["type"], "complete_workflow");
21337            assert!(!commands
21338                .iter()
21339                .any(|command| command["type"] == "start_child_workflow"));
21340            let output =
21341                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21342            assert_eq!(output["parent_workflow_id"], "wf-parent");
21343            assert_eq!(output["parent_run_id"], "run-parent");
21344            assert_eq!(output["child_workflow_id"], "wf-child");
21345            assert_eq!(output["child_run_id"], "run-child");
21346            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21347        }
21348    }
21349
21350    #[test]
21351    fn typed_child_arguments_and_results_survive_replay() {
21352        let client = Client::new("http://127.0.0.1:8080").expect("client");
21353        let mut worker = Worker::new(client, "rust-parent-workers");
21354        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21355            let child = ctx
21356                .start_child_workflow_avro_value(
21357                    "python.typed-child",
21358                    ChildWorkflowOptions::new("python-workers"),
21359                    AvroValue::Array(vec![typed_fidelity_probe()]),
21360                )
21361                .await?;
21362            Ok(child.result)
21363        });
21364
21365        let initial = worker
21366            .execute_workflow_task(workflow_task(
21367                "rust.typed-parent",
21368                Vec::new(),
21369                DEFAULT_CODEC,
21370            ))
21371            .expect("typed child start");
21372        assert_eq!(initial[0]["type"], "start_child_workflow");
21373        assert_eq!(
21374            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21375                .expect("typed child arguments"),
21376            AvroValue::Array(vec![typed_fidelity_probe()])
21377        );
21378
21379        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21380            .expect("typed child result");
21381        let task = workflow_task(
21382            "rust.typed-parent",
21383            vec![
21384                history_event(
21385                    "ChildWorkflowScheduled",
21386                    json!({
21387                        "sequence": 1,
21388                        "child_call_id": "call-typed",
21389                        "child_workflow_instance_id": "wf-child",
21390                        "child_workflow_run_id": "run-child",
21391                        "child_workflow_type": "python.typed-child",
21392                    }),
21393                ),
21394                history_event(
21395                    "ChildRunCompleted",
21396                    json!({
21397                        "sequence": 1,
21398                        "child_call_id": "call-typed",
21399                        "child_workflow_instance_id": "wf-child",
21400                        "child_workflow_run_id": "run-child",
21401                        "child_workflow_type": "python.typed-child",
21402                        "payload_codec": DEFAULT_CODEC,
21403                        "result": result,
21404                    }),
21405                ),
21406            ],
21407            DEFAULT_CODEC,
21408        );
21409
21410        let commands = worker
21411            .execute_workflow_task(task)
21412            .expect("typed child replay");
21413        assert_eq!(commands[0]["type"], "complete_workflow");
21414        assert_eq!(
21415            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21416                .expect("typed parent result"),
21417            typed_fidelity_probe()
21418        );
21419    }
21420
21421    #[test]
21422    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21423        let worker = child_parent_worker();
21424        let mut task = child_parent_task("unused", Value::Null);
21425        task.history_events.truncate(1);
21426        task.total_history_events = Some(1);
21427
21428        for _redelivery_or_restart in 0..2 {
21429            let commands = worker
21430                .execute_workflow_task(task.clone())
21431                .expect("recorded child remains pending");
21432            assert!(
21433                commands.is_empty(),
21434                "recorded pending child must not be started again"
21435            );
21436        }
21437    }
21438
21439    #[test]
21440    fn child_cancellation_becomes_stable_parent_failure_command() {
21441        let worker = child_parent_worker();
21442        let task = child_parent_task(
21443            "ChildRunCancelled",
21444            json!({
21445                "sequence": 1,
21446                "child_workflow_instance_id": "wf-child",
21447                "child_workflow_run_id": "run-child",
21448                "child_workflow_type": "python.child",
21449                "failure_id": "failure-child",
21450                "failure_category": "cancelled",
21451                "message": "cancelled by parent-close policy",
21452            }),
21453        );
21454
21455        let commands = worker
21456            .execute_workflow_task(task)
21457            .expect("parent settlement");
21458        assert_eq!(commands.len(), 1);
21459        assert_eq!(commands[0]["type"], "fail_workflow");
21460        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21461        assert_eq!(
21462            commands[0]["exception"]["properties"]["reason"],
21463            "cancelled"
21464        );
21465        assert_eq!(
21466            commands[0]["exception"]["properties"]["child_workflow_run_id"],
21467            "run-child"
21468        );
21469    }
21470
21471    #[test]
21472    fn workflow_can_handle_typed_child_failure() {
21473        let client = Client::new("http://127.0.0.1:8080").expect("client");
21474        let mut worker = Worker::new(client, "rust-parent-workers");
21475        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21476            match ctx
21477                .start_child_workflow(
21478                    "python.child",
21479                    ChildWorkflowOptions::new("python-child-workers"),
21480                    json!([]),
21481                )
21482                .await
21483            {
21484                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21485                    "reason": failure.reason,
21486                    "failure_id": failure.failure_id,
21487                    "exception_class": failure.exception_class,
21488                    "child_run_id": failure.child_workflow_run_id,
21489                })),
21490                Err(error) => Err(error),
21491                Ok(_) => Err(Error::WorkerLoop(
21492                    "child unexpectedly succeeded".to_string(),
21493                )),
21494            }
21495        });
21496        let mut task = child_parent_task(
21497            "ChildRunFailed",
21498            json!({
21499                "sequence": 1,
21500                "child_workflow_instance_id": "wf-child",
21501                "child_workflow_run_id": "run-child",
21502                "child_workflow_type": "python.child",
21503                "failure_id": "failure-child",
21504                "failure_category": "child_workflow",
21505                "message": "payment rejected",
21506                "exception": {
21507                    "type": "PaymentRejected",
21508                    "class": "payments.PaymentRejected",
21509                    "message": "payment rejected"
21510                }
21511            }),
21512        );
21513        task.workflow_type = "rust.handled-parent".to_string();
21514
21515        let commands = worker.execute_workflow_task(task).expect("handled failure");
21516        assert_eq!(commands[0]["type"], "complete_workflow");
21517        let output =
21518            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21519        assert_eq!(output["reason"], "child_workflow");
21520        assert_eq!(output["failure_id"], "failure-child");
21521        assert_eq!(output["exception_class"], "payments.PaymentRejected");
21522        assert_eq!(output["child_run_id"], "run-child");
21523    }
21524
21525    #[test]
21526    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21527        let client = Client::new("http://127.0.0.1:8080").expect("client");
21528        let mut worker = Worker::new(client, "rust-workers");
21529
21530        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21531            let signal = ctx.wait_signal("start").await?;
21532            let name = signal
21533                .first()
21534                .and_then(|value| value.as_str())
21535                .unwrap_or("world");
21536            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21537            Ok(json!({
21538                "greeting": greeting,
21539                "language": "rust"
21540            }))
21541        });
21542
21543        let signal_arguments =
21544            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21545        let task = WorkflowTask {
21546            task_id: "wft-rust-signal-1".to_string(),
21547            workflow_command_id: None,
21548            workflow_id: Some("wf-rust-hello".to_string()),
21549            run_id: Some("run-rust-hello".to_string()),
21550            workflow_type: "rust.hello_workflow".to_string(),
21551            cancel_requested: false,
21552            payload_codec: DEFAULT_CODEC.to_string(),
21553            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21554            history_events: vec![HistoryEvent {
21555                event_type: "SignalReceived".to_string(),
21556                payload: json!({
21557                    "signal_id": "sig-rust-1",
21558                    "signal_name": "start"
21559                }),
21560                raw: HashMap::new(),
21561            }],
21562            total_history_events: Some(1),
21563            history_size_bytes: None,
21564            continue_as_new_recommended: None,
21565            history_budget_pressure: None,
21566            next_history_page_token: None,
21567            workflow_task_attempt: 1,
21568            workflow_signal_id: Some("sig-rust-1".to_string()),
21569            signal_name: Some("start".to_string()),
21570            signal_arguments: Some(signal_arguments),
21571            workflow_update_id: None,
21572            update_name: None,
21573            lease_owner: Some("rust-worker".to_string()),
21574        };
21575
21576        let commands = worker.execute_workflow_task(task).expect("workflow task");
21577
21578        assert_eq!(commands.len(), 1);
21579        assert_eq!(commands[0]["type"], "schedule_activity");
21580        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21581        assert_eq!(
21582            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21583            json!(["Rust"])
21584        );
21585    }
21586
21587    #[test]
21588    fn workflow_task_appends_paginated_history_events() {
21589        let mut task = WorkflowTask {
21590            task_id: "wft-rust-pages-1".to_string(),
21591            workflow_command_id: None,
21592            workflow_id: Some("wf-rust-pages".to_string()),
21593            run_id: Some("run-rust-pages".to_string()),
21594            workflow_type: "rust.hello_workflow".to_string(),
21595            cancel_requested: false,
21596            payload_codec: DEFAULT_CODEC.to_string(),
21597            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21598            history_events: vec![HistoryEvent {
21599                event_type: "WorkflowStarted".to_string(),
21600                payload: json!({}),
21601                raw: HashMap::new(),
21602            }],
21603            total_history_events: Some(3),
21604            history_size_bytes: None,
21605            continue_as_new_recommended: None,
21606            history_budget_pressure: None,
21607            next_history_page_token: Some("MQ==".to_string()),
21608            workflow_task_attempt: 1,
21609            workflow_signal_id: None,
21610            signal_name: None,
21611            signal_arguments: None,
21612            workflow_update_id: None,
21613            update_name: None,
21614            lease_owner: Some("rust-worker".to_string()),
21615        };
21616
21617        task.append_history_page(WorkflowTaskHistoryPage {
21618            history_events: vec![
21619                HistoryEvent {
21620                    event_type: "SignalReceived".to_string(),
21621                    payload: json!({
21622                        "signal_id": "sig-rust-1",
21623                        "signal_name": "start",
21624                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21625                            .expect("signal arguments")
21626                    }),
21627                    raw: HashMap::new(),
21628                },
21629                HistoryEvent {
21630                    event_type: "MarkerRecorded".to_string(),
21631                    payload: json!({"sequence": 3}),
21632                    raw: HashMap::new(),
21633                },
21634            ],
21635            total_history_events: Some(3),
21636            next_history_page_token: None,
21637        });
21638
21639        assert_eq!(task.history_events.len(), 3);
21640        assert_eq!(task.total_history_events, Some(3));
21641        assert_eq!(task.next_history_page_token, None);
21642
21643        let signal = task
21644            .history_events
21645            .iter()
21646            .find(|event| event.event_type == "SignalReceived")
21647            .expect("signal event");
21648        assert_eq!(
21649            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21650            vec![AvroValue::String("Rust".to_string())]
21651        );
21652    }
21653
21654    #[tokio::test]
21655    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21656        let client = Client::new("http://127.0.0.1:8080").expect("client");
21657        let mut worker = Worker::new(client, "rust-workers");
21658        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21659        worker.register_query("counter", "current", |ctx, _args| async move {
21660            let mut count = 0_i64;
21661            for signal in ctx.signal_events() {
21662                let value = signal
21663                    .arguments
21664                    .first()
21665                    .and_then(Value::as_i64)
21666                    .unwrap_or_default();
21667                match signal.name.as_str() {
21668                    "increment" => count += value,
21669                    "set" => count = value,
21670                    _ => {}
21671                }
21672            }
21673            Ok(json!(count))
21674        });
21675
21676        let task = QueryTask {
21677            query_task_id: "query-rust-counter".to_string(),
21678            query_task_attempt: 1,
21679            lease_owner: Some("rust-worker".to_string()),
21680            workflow_id: Some("counter-1".to_string()),
21681            run_id: Some("run-counter-1".to_string()),
21682            workflow_type: "counter".to_string(),
21683            query_name: "current".to_string(),
21684            payload_codec: DEFAULT_CODEC.to_string(),
21685            workflow_arguments: Some(
21686                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21687            ),
21688            query_arguments: Some(
21689                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21690            ),
21691            history_events: vec![
21692                HistoryEvent {
21693                    event_type: "SignalReceived".to_string(),
21694                    payload: json!({
21695                        "signal_id": "php-signal-1",
21696                        "signal_name": "increment",
21697                        "workflow_sequence": 1,
21698                        "payload_codec": DEFAULT_CODEC,
21699                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21700                    }),
21701                    raw: HashMap::new(),
21702                },
21703                HistoryEvent {
21704                    event_type: "SignalReceived".to_string(),
21705                    payload: json!({
21706                        "signal_id": "python-signal-2",
21707                        "signal_name": "increment",
21708                        "workflow_sequence": 2,
21709                        "payload_codec": DEFAULT_CODEC,
21710                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21711                    }),
21712                    raw: HashMap::new(),
21713                },
21714                HistoryEvent {
21715                    event_type: "SignalReceived".to_string(),
21716                    payload: json!({
21717                        "signal_id": "rust-signal-3",
21718                        "signal_name": "set",
21719                        "workflow_sequence": 3,
21720                        "payload_codec": DEFAULT_CODEC,
21721                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21722                    }),
21723                    raw: HashMap::new(),
21724                },
21725            ],
21726            history_export: None,
21727            run_status: Some("completed".to_string()),
21728        };
21729
21730        let result = worker.execute_query_task(task).await.expect("query result");
21731        assert_eq!(result.into_json().expect("query projection"), json!(0));
21732    }
21733
21734    #[tokio::test]
21735    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21736        let worker = replay_counter_worker();
21737        let running_history = json!([
21738            {
21739                "type": "ActivityCompleted",
21740                "payload": {
21741                    "sequence": 1,
21742                    "activity_type": "load-counter",
21743                    "payload_codec": DEFAULT_CODEC,
21744                    "result": fixture_envelope(json!("loaded"))
21745                }
21746            },
21747            {
21748                "type": "SignalWaitOpened",
21749                "payload": {
21750                    "sequence": 3,
21751                    "signal_name": "increment"
21752                }
21753            },
21754            {
21755                "type": "SignalReceived",
21756                "payload": {
21757                    "signal_id": "signal-3",
21758                    "signal_name": "increment",
21759                    "workflow_sequence": 2,
21760                    "payload_codec": DEFAULT_CODEC,
21761                    "arguments": fixture_envelope(json!([3]))
21762                }
21763            },
21764            {
21765                "type": "SignalApplied",
21766                "payload": {
21767                    "sequence": 3,
21768                    "signal_id": "signal-3",
21769                    "signal_name": "increment",
21770                    "payload_codec": DEFAULT_CODEC,
21771                    "value": fixture_envelope(json!([3]))
21772                }
21773            }
21774        ]);
21775
21776        let running = worker
21777            .execute_query_task(replay_counter_query(
21778                "current",
21779                running_history.clone(),
21780                "running",
21781            ))
21782            .await
21783            .expect("running replay query");
21784        assert_eq!(
21785            running.clone().into_json().expect("query projection"),
21786            json!({"loaded": "loaded", "count": 3, "finished": false})
21787        );
21788
21789        let detached = worker
21790            .execute_query_task(replay_counter_query(
21791                "detached-mutation",
21792                running_history.clone(),
21793                "running",
21794            ))
21795            .await
21796            .expect("query mutates only its detached state clone");
21797        assert_eq!(detached.into_json().expect("query projection"), json!(999));
21798        let failed = worker
21799            .execute_query_task(replay_counter_query(
21800                "failed-mutation",
21801                running_history.clone(),
21802                "running",
21803            ))
21804            .await
21805            .expect_err("failed query");
21806        assert_eq!(failed.reason, "query_rejected");
21807        let unchanged = worker
21808            .execute_query_task(replay_counter_query("current", running_history, "running"))
21809            .await
21810            .expect("later query reconstructs unchanged state");
21811        assert_eq!(unchanged, running);
21812
21813        let restarted_worker = replay_counter_worker();
21814        let empty_arguments = fixture_envelope(json!([]));
21815        let loaded_result = fixture_envelope(json!("loaded"));
21816        let signal_three = fixture_blob(json!([3]));
21817        let signal_five = fixture_blob(json!([5]));
21818        let restarted_task: QueryTask = serde_json::from_value(json!({
21819            "query_task_id": "query-after-restart",
21820            "workflow_id": "counter-1",
21821            "run_id": "run-counter-1",
21822            "workflow_type": "replay-counter",
21823            "query_name": "current",
21824            "payload_codec": DEFAULT_CODEC,
21825            "workflow_arguments": empty_arguments.clone(),
21826            "query_arguments": empty_arguments,
21827            "history_events": [],
21828            "history_export": {
21829                "payloads": {"codec": DEFAULT_CODEC},
21830                "history_events": [
21831                    {
21832                        "type": "ActivityCompleted",
21833                        "payload": {
21834                            "sequence": 1,
21835                            "activity_type": "load-counter",
21836                            "payload_codec": DEFAULT_CODEC,
21837                            "result": null
21838                        }
21839                    },
21840                    {
21841                        "type": "SignalWaitOpened",
21842                        "payload": {
21843                            "sequence": 3,
21844                            "signal_name": "increment"
21845                        }
21846                    },
21847                    {
21848                        "type": "SignalReceived",
21849                        "payload": {
21850                            "signal_id": "signal-3",
21851                            "signal_name": "increment",
21852                            "workflow_sequence": 2
21853                        }
21854                    },
21855                    {
21856                        "type": "SignalApplied",
21857                        "payload": {
21858                            "sequence": 3,
21859                            "signal_id": "signal-3",
21860                            "signal_name": "increment"
21861                        }
21862                    },
21863                    {
21864                        "type": "SignalWaitOpened",
21865                        "payload": {
21866                            "sequence": 5,
21867                            "signal_name": "increment"
21868                        }
21869                    },
21870                    {
21871                        "type": "SignalReceived",
21872                        "payload": {
21873                            "signal_id": "signal-5",
21874                            "signal_name": "increment",
21875                            "workflow_sequence": 4
21876                        }
21877                    },
21878                    {
21879                        "type": "SignalApplied",
21880                        "payload": {
21881                            "sequence": 5,
21882                            "signal_id": "signal-5",
21883                            "signal_name": "increment"
21884                        }
21885                    }
21886                ],
21887                "activities": [{
21888                    "sequence": 1,
21889                    "activity_type": "load-counter",
21890                    "payload_codec": DEFAULT_CODEC,
21891                    "result": loaded_result
21892                }],
21893                "signals": [
21894                    {
21895                        "id": "signal-3",
21896                        "name": "increment",
21897                        "workflow_sequence": 2,
21898                        "payload_codec": DEFAULT_CODEC,
21899                        "arguments": signal_three
21900                    },
21901                    {
21902                        "id": "signal-5",
21903                        "name": "increment",
21904                        "workflow_sequence": 4,
21905                        "payload_codec": DEFAULT_CODEC,
21906                        "arguments": signal_five
21907                    }
21908                ]
21909            },
21910            "run_status": "completed"
21911        }))
21912        .expect("cold replay query task");
21913        let completed = restarted_worker
21914            .execute_query_task(restarted_task)
21915            .await
21916            .expect("completed cold replay query");
21917        assert_eq!(
21918            completed.into_json().expect("query projection"),
21919            json!({"loaded": "loaded", "count": 8, "finished": true})
21920        );
21921    }
21922
21923    #[tokio::test]
21924    async fn replayed_query_replay_failures_are_machine_readable() {
21925        let worker = replay_counter_worker();
21926        let task = replay_counter_query(
21927            "current",
21928            json!([{
21929                "type": "ActivityCompleted",
21930                "payload": {
21931                    "sequence": 1,
21932                    "payload_codec": DEFAULT_CODEC,
21933                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21934                }
21935            }]),
21936            "running",
21937        );
21938        let failure = worker
21939            .execute_query_task(task)
21940            .await
21941            .expect_err("invalid replay history payload");
21942        assert_eq!(failure.reason, "query_payload_decode_failed");
21943        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21944        assert!(failure.message.contains("invalid_payload_framing"));
21945    }
21946
21947    #[tokio::test]
21948    async fn query_task_restores_compact_history_from_export() {
21949        let client = Client::new("http://127.0.0.1:8080").expect("client");
21950        let mut worker = Worker::new(client, "rust-workers");
21951        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21952        worker.register_query("counter", "current", |ctx, _args| async move {
21953            Ok(json!(ctx.signals("increment")[0][0]))
21954        });
21955        let empty_arguments = fixture_envelope(json!([]));
21956        let exported_signal = fixture_blob(json!([9]));
21957        let task: QueryTask = serde_json::from_value(json!({
21958            "query_task_id": "query-export",
21959            "workflow_type": "counter",
21960            "query_name": "current",
21961            "payload_codec": DEFAULT_CODEC,
21962            "workflow_arguments": empty_arguments.clone(),
21963            "query_arguments": empty_arguments,
21964            "history_events": [],
21965            "history_export": {
21966                "payloads": {"codec": DEFAULT_CODEC},
21967                "history_events": [{
21968                    "type": "SignalReceived",
21969                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21970                }],
21971                "signals": [{
21972                    "id": "signal-export",
21973                    "name": "increment",
21974                    "status": "applied",
21975                    "workflow_sequence": 1,
21976                    "payload_codec": DEFAULT_CODEC,
21977                    "arguments": exported_signal
21978                }]
21979            }
21980        }))
21981        .expect("query task");
21982
21983        let result = worker.execute_query_task(task).await.expect("query result");
21984        assert_eq!(result.into_json().expect("query projection"), json!(9));
21985    }
21986
21987    #[tokio::test]
21988    async fn query_task_failures_have_stable_reasons() {
21989        let client = Client::new("http://127.0.0.1:8080").expect("client");
21990        let mut worker = Worker::new(client, "rust-workers");
21991        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21992        worker.register_query(
21993            "counter",
21994            "current",
21995            |_ctx, _args| async move { Ok(json!(0)) },
21996        );
21997
21998        let base_task = QueryTask {
21999            query_task_id: "query-errors".to_string(),
22000            query_task_attempt: 1,
22001            lease_owner: None,
22002            workflow_id: Some("counter-errors".to_string()),
22003            run_id: Some("run-errors".to_string()),
22004            workflow_type: "counter".to_string(),
22005            query_name: "missing".to_string(),
22006            payload_codec: DEFAULT_CODEC.to_string(),
22007            workflow_arguments: Some(fixture_envelope(json!([]))),
22008            query_arguments: Some(fixture_envelope(json!([]))),
22009            history_events: Vec::new(),
22010            history_export: None,
22011            run_status: Some("running".to_string()),
22012        };
22013
22014        let unknown = worker
22015            .execute_query_task(base_task.clone())
22016            .await
22017            .expect_err("unknown query");
22018        assert_eq!(unknown.reason, "rejected_unknown_query");
22019
22020        let mut malformed = base_task;
22021        malformed.query_name = "current".to_string();
22022        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
22023        let malformed = worker
22024            .execute_query_task(malformed)
22025            .await
22026            .expect_err("malformed payload");
22027        assert_eq!(malformed.reason, "query_payload_decode_failed");
22028
22029        let client = Client::new("http://127.0.0.1:8080").expect("client");
22030        let mut unavailable_worker = Worker::new(client, "rust-workers");
22031        unavailable_worker
22032            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22033        let empty_arguments = fixture_envelope(json!([]));
22034        let unavailable_task: QueryTask = serde_json::from_value(json!({
22035            "query_task_id": "query-unavailable",
22036            "workflow_type": "counter",
22037            "query_name": "current",
22038            "payload_codec": DEFAULT_CODEC,
22039            "workflow_arguments": empty_arguments.clone(),
22040            "query_arguments": empty_arguments
22041        }))
22042        .expect("query task");
22043        let unavailable = unavailable_worker
22044            .execute_query_task(unavailable_task)
22045            .await
22046            .expect_err("query handler unavailable");
22047        assert_eq!(unavailable.reason, "query_handler_unavailable");
22048    }
22049
22050    #[tokio::test]
22051    async fn client_query_decodes_result_and_typed_failure() {
22052        let server = MockWorkerServer::start();
22053        let client = Client::builder(server.base_url())
22054            .timeout(Duration::from_secs(2))
22055            .build()
22056            .expect("client");
22057
22058        let result = client
22059            .query_workflow("counter-1", "current", json!([]))
22060            .await
22061            .expect("query result");
22062        assert_eq!(result, json!({"count": 8}));
22063
22064        let error = client
22065            .query_workflow("counter-1", "missing", json!([]))
22066            .await
22067            .expect_err("unknown query");
22068        let Error::QueryFailed(failure) = error else {
22069            panic!("expected typed query failure");
22070        };
22071        assert_eq!(failure.status, 404);
22072        assert_eq!(failure.reason, "rejected_unknown_query");
22073    }
22074
22075    #[tokio::test]
22076    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
22077        let server = MockWorkerServer::start();
22078        let client = Client::builder(server.base_url())
22079            .timeout(Duration::from_secs(2))
22080            .build()
22081            .expect("client");
22082        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
22083
22084        client
22085            .start_workflow(
22086                "typed.echo",
22087                "rust-workers",
22088                "typed-start",
22089                arguments.clone(),
22090            )
22091            .await
22092            .expect("typed workflow start");
22093        assert_eq!(
22094            decode_wire_avro_value(
22095                &server.request_body("/api/workflows")["input"],
22096                DEFAULT_CODEC,
22097            )
22098            .expect("typed start input"),
22099            arguments
22100        );
22101
22102        client
22103            .signal_workflow("typed-1", "changed", arguments.clone())
22104            .await
22105            .expect("typed signal");
22106        assert_eq!(
22107            decode_wire_avro_value(
22108                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
22109                DEFAULT_CODEC,
22110            )
22111            .expect("typed signal input"),
22112            arguments
22113        );
22114
22115        assert_eq!(
22116            client
22117                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22118                .await
22119                .expect("typed query"),
22120            typed_fidelity_probe()
22121        );
22122        assert_eq!(
22123            decode_wire_avro_value(
22124                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22125                DEFAULT_CODEC,
22126            )
22127            .expect("typed query input"),
22128            arguments
22129        );
22130
22131        assert_eq!(
22132            client
22133                .update_workflow_avro_value(
22134                    "typed-1",
22135                    "replace",
22136                    arguments.clone(),
22137                    Some("typed-request"),
22138                )
22139                .await
22140                .expect("typed update"),
22141            typed_fidelity_probe()
22142        );
22143        let update = server.request_body("/api/workflows/typed-1/update/replace");
22144        assert_eq!(update["request_id"], "typed-request");
22145        assert_eq!(
22146            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22147            arguments
22148        );
22149
22150        let handle = WorkflowHandle {
22151            client: client.clone(),
22152            workflow_id: "typed-1".to_string(),
22153            run_id: Some("run-typed-1".to_string()),
22154            workflow_type: "typed.echo".to_string(),
22155        };
22156        assert_eq!(
22157            handle
22158                .result_avro_value(WorkflowResultOptions::default())
22159                .await
22160                .expect("typed workflow result"),
22161            typed_fidelity_probe()
22162        );
22163
22164        client
22165            .complete_activity_task(
22166                "activity-typed",
22167                "attempt-typed",
22168                "rust-worker",
22169                typed_fidelity_probe(),
22170                DEFAULT_CODEC,
22171            )
22172            .await
22173            .expect("typed activity completion");
22174        assert_eq!(
22175            decode_wire_avro_value(
22176                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22177                    ["result"],
22178                DEFAULT_CODEC,
22179            )
22180            .expect("typed activity result"),
22181            typed_fidelity_probe()
22182        );
22183        client
22184            .fail_activity_task(
22185                "activity-typed",
22186                "attempt-typed",
22187                "rust-worker",
22188                "typed failure",
22189                true,
22190            )
22191            .await
22192            .expect("activity failure");
22193    }
22194
22195    #[tokio::test]
22196    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22197        let server = MockWorkerServer::start();
22198        let client = Client::builder(server.base_url())
22199            .timeout(Duration::from_secs(2))
22200            .build()
22201            .expect("client");
22202
22203        let options = WorkflowCommandOptions::new()
22204            .reason("cleanup requested")
22205            .request_id("cancel-17");
22206        let cancelled = client
22207            .cancel_workflow("wf-lifecycle", options)
22208            .await
22209            .expect("instance cancellation");
22210        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22211        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22212        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22213        assert_eq!(
22214            server.request_body("/api/workflows/wf-lifecycle/cancel"),
22215            json!({"reason":"cleanup requested","request_id":"cancel-17"})
22216        );
22217
22218        let terminated = client
22219            .terminate_workflow(
22220                "wf-lifecycle",
22221                WorkflowCommandOptions::new().reason("forced stop"),
22222            )
22223            .await
22224            .expect("instance termination");
22225        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22226        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22227
22228        client
22229            .cancel_workflow_run(
22230                "wf-lifecycle",
22231                "run-current",
22232                WorkflowCommandOptions::default(),
22233            )
22234            .await
22235            .expect("selected run cancellation");
22236        client
22237            .terminate_workflow_run(
22238                "wf-lifecycle",
22239                "run-current",
22240                WorkflowCommandOptions::default(),
22241            )
22242            .await
22243            .expect("selected run termination");
22244
22245        for (command, error) in [
22246            (
22247                WorkflowCommandKind::Cancel,
22248                client
22249                    .cancel_workflow_run(
22250                        "wf-lifecycle",
22251                        "run-stale",
22252                        WorkflowCommandOptions::default(),
22253                    )
22254                    .await
22255                    .expect_err("stale cancellation must be rejected"),
22256            ),
22257            (
22258                WorkflowCommandKind::Terminate,
22259                client
22260                    .terminate_workflow_run(
22261                        "wf-lifecycle",
22262                        "run-stale",
22263                        WorkflowCommandOptions::default(),
22264                    )
22265                    .await
22266                    .expect_err("stale termination must be rejected"),
22267            ),
22268        ] {
22269            let Error::WorkflowCommandRejected(rejection) = error else {
22270                panic!("expected typed command rejection");
22271            };
22272            assert_eq!(rejection.command, command);
22273            assert_eq!(rejection.status, 409);
22274            assert_eq!(rejection.reason, "historical_run_command_rejected");
22275            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22276            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22277        }
22278    }
22279
22280    #[tokio::test]
22281    async fn workflow_start_options_send_server_enforced_deadlines() {
22282        let server = MockWorkerServer::start();
22283        let client = Client::builder(server.base_url())
22284            .timeout(Duration::from_secs(2))
22285            .build()
22286            .expect("client");
22287
22288        let handle = client
22289            .start_workflow_with_options(
22290                "rust.timeout",
22291                "rust-timeouts",
22292                "wf-start-options",
22293                WorkflowStartOptions::new()
22294                    .execution_timeout_seconds(30)
22295                    .run_timeout_seconds(1),
22296                json!([]),
22297            )
22298            .await
22299            .expect("workflow start");
22300
22301        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22302        let body = server.request_body("/api/workflows");
22303        assert_eq!(body["execution_timeout_seconds"], 30);
22304        assert_eq!(body["run_timeout_seconds"], 1);
22305
22306        let invalid = client
22307            .start_workflow_with_options(
22308                "rust.timeout",
22309                "rust-timeouts",
22310                "wf-invalid-options",
22311                WorkflowStartOptions::new()
22312                    .execution_timeout_seconds(1)
22313                    .run_timeout_seconds(2),
22314                json!([]),
22315            )
22316            .await
22317            .expect_err("invalid deadline ordering");
22318        assert!(invalid
22319            .to_string()
22320            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22321    }
22322
22323    #[tokio::test]
22324    async fn workflow_result_returns_each_typed_terminal_outcome() {
22325        let server = MockWorkerServer::start();
22326        let client = Client::builder(server.base_url())
22327            .timeout(Duration::from_secs(2))
22328            .build()
22329            .expect("client");
22330        let options = WorkflowResultOptions {
22331            poll_interval: Duration::ZERO,
22332            timeout: Duration::from_secs(1),
22333        };
22334
22335        let failed = WorkflowHandle {
22336            client: client.clone(),
22337            workflow_id: "wf-failed".to_string(),
22338            run_id: Some("run-failed".to_string()),
22339            workflow_type: "failure".to_string(),
22340        }
22341        .result(options)
22342        .await
22343        .expect_err("failed outcome");
22344        let Error::WorkflowFailed(failure) = failed else {
22345            panic!("expected WorkflowFailed");
22346        };
22347        assert_eq!(failure.workflow_id, "wf-failed");
22348        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22349        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22350        assert_eq!(failure.failure_category.as_deref(), Some("application"));
22351        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22352        assert_eq!(
22353            failure.exception_class.as_deref(),
22354            Some("billing::PaymentError")
22355        );
22356        assert_eq!(failure.non_retryable, Some(true));
22357
22358        for (workflow_id, expected_kind, expected_reason) in [
22359            (
22360                "wf-cancelled",
22361                WorkflowTerminalKind::Cancelled,
22362                "cleanup requested",
22363            ),
22364            (
22365                "wf-terminated",
22366                WorkflowTerminalKind::Terminated,
22367                "forced stop",
22368            ),
22369            (
22370                "wf-timed-out",
22371                WorkflowTerminalKind::TimedOut,
22372                "run_timeout",
22373            ),
22374        ] {
22375            let error = WorkflowHandle {
22376                client: client.clone(),
22377                workflow_id: workflow_id.to_string(),
22378                run_id: None,
22379                workflow_type: "terminal".to_string(),
22380            }
22381            .result(options)
22382            .await
22383            .expect_err("typed terminal outcome");
22384            let outcome = match error {
22385                Error::WorkflowCancelled(outcome) => outcome,
22386                Error::WorkflowTerminated(outcome) => outcome,
22387                Error::WorkflowTimedOut(outcome) => outcome,
22388                other => panic!("unexpected terminal error: {other}"),
22389            };
22390            assert_eq!(outcome.kind, expected_kind);
22391            assert_eq!(outcome.workflow_id, workflow_id);
22392            assert_eq!(outcome.reason, expected_reason);
22393        }
22394
22395        let wait_timeout = WorkflowHandle {
22396            client,
22397            workflow_id: "wf-waiting".to_string(),
22398            run_id: Some("run-waiting".to_string()),
22399            workflow_type: "waiting".to_string(),
22400        }
22401        .result(WorkflowResultOptions {
22402            poll_interval: Duration::ZERO,
22403            timeout: Duration::ZERO,
22404        })
22405        .await
22406        .expect_err("client wait timeout");
22407        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22408            panic!("expected typed client timeout");
22409        };
22410        assert_eq!(timeout.reason, "result_wait_timeout");
22411        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22412        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22413    }
22414
22415    #[tokio::test]
22416    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22417        let server = MockWorkerServer::start();
22418        let client = Client::builder(server.base_url())
22419            .timeout(Duration::from_secs(2))
22420            .build()
22421            .expect("client");
22422
22423        let handle = WorkflowHandle {
22424            client,
22425            workflow_id: "wf-selected".to_string(),
22426            run_id: Some("run-selected".to_string()),
22427            workflow_type: "selected".to_string(),
22428        };
22429        let options = WorkflowResultOptions {
22430            poll_interval: Duration::ZERO,
22431            timeout: Duration::from_secs(1),
22432        };
22433
22434        let current = handle
22435            .result(options)
22436            .await
22437            .expect("instance result follows the current run");
22438        assert_eq!(current, json!("current run output"));
22439
22440        let error = handle
22441            .result_selected_run(options)
22442            .await
22443            .expect_err("the selected run is cancelled even though the current run completed");
22444
22445        let Error::WorkflowCancelled(outcome) = error else {
22446            panic!("expected selected run cancellation");
22447        };
22448        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22449        assert_eq!(outcome.reason, "selected run cancelled");
22450        assert_eq!(
22451            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22452            1
22453        );
22454        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22455    }
22456
22457    #[tokio::test]
22458    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22459        let server = MockWorkerServer::draining_polls();
22460        let client = Client::builder(server.base_url())
22461            .timeout(Duration::from_secs(2))
22462            .build()
22463            .expect("client");
22464
22465        let workflow = client
22466            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22467            .await
22468            .expect("workflow drain response");
22469        let activity = client
22470            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22471            .await
22472            .expect("activity drain response");
22473        let query = client
22474            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22475            .await
22476            .expect("query drain response");
22477
22478        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22479            assert_eq!(
22480                outcome,
22481                WorkerPollOutcome::Stop {
22482                    poll_status: Some("draining".to_string()),
22483                    reason: Some("worker_draining".to_string()),
22484                }
22485            );
22486        }
22487
22488        assert!(client
22489            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22490            .await
22491            .expect("compatibility poll")
22492            .is_none());
22493    }
22494
22495    #[tokio::test]
22496    async fn managed_worker_honors_drain_stop_for_every_task_family() {
22497        let server = MockWorkerServer::draining_polls();
22498        let client = Client::builder(server.base_url())
22499            .timeout(Duration::from_secs(2))
22500            .build()
22501            .expect("client");
22502
22503        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22504            .worker_id("draining-workflow-worker")
22505            .poll_timeout(Duration::ZERO);
22506        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22507        workflow_worker
22508            .run()
22509            .await
22510            .expect("workflow drain is a clean stop");
22511
22512        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22513            .worker_id("draining-activity-worker")
22514            .poll_timeout(Duration::ZERO);
22515        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22516        activity_worker
22517            .run()
22518            .await
22519            .expect("activity drain is a clean stop");
22520
22521        let mut query_worker = Worker::new(client, "rust-workers")
22522            .worker_id("draining-query-worker")
22523            .poll_timeout(Duration::ZERO);
22524        query_worker.register_query("counter", "current", |_ctx, _args| async {
22525            Ok(Value::Null)
22526        });
22527        query_worker
22528            .run()
22529            .await
22530            .expect("query drain is a clean stop");
22531    }
22532
22533    #[tokio::test]
22534    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22535        let server = MockWorkerServer::start();
22536        let client = Client::builder(server.base_url())
22537            .timeout(Duration::from_secs(2))
22538            .build()
22539            .expect("client");
22540
22541        let heartbeat = client
22542            .heartbeat_activity_task(
22543                "activity-cancel",
22544                "attempt-cancel",
22545                "rust-worker",
22546                typed_fidelity_probe(),
22547            )
22548            .await
22549            .expect("cancellation heartbeat");
22550        assert!(heartbeat.cancel_requested);
22551        assert!(heartbeat.should_stop());
22552        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22553        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22554        let heartbeat_body =
22555            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22556        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22557        assert_eq!(
22558            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22559                .expect("typed heartbeat details"),
22560            typed_fidelity_probe()
22561        );
22562
22563        let error = client
22564            .complete_activity_task(
22565                "activity-cancel",
22566                "attempt-cancel",
22567                "rust-worker",
22568                json!({"late":true}),
22569                DEFAULT_CODEC,
22570            )
22571            .await
22572            .expect_err("late completion must be refused");
22573        assert!(activity_task_rejection_is_final(&error));
22574        let Error::ActivityTaskRejected(rejection) = error else {
22575            panic!("expected typed activity rejection");
22576        };
22577        assert_eq!(rejection.status, 409);
22578        assert_eq!(rejection.reason, "run_cancelled");
22579        assert!(rejection.cancel_requested);
22580        assert_eq!(rejection.can_continue, Some(false));
22581    }
22582
22583    #[tokio::test]
22584    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22585        let server = MockWorkerServer::cancelled_activity();
22586        let client = Client::builder(server.base_url())
22587            .timeout(Duration::from_secs(2))
22588            .build()
22589            .expect("client");
22590        let cancellation_observed = Arc::new(AtomicBool::new(false));
22591        let observed = Arc::clone(&cancellation_observed);
22592        let mut worker = Worker::new(client.clone(), "rust-workers")
22593            .worker_id("rust-cancel-worker")
22594            .poll_timeout(Duration::from_millis(10));
22595        worker.register_activity("cancel-aware", move |ctx, _args| {
22596            let observed = Arc::clone(&observed);
22597            async move {
22598                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22599                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22600                Ok(json!({"late":"completion"}))
22601            }
22602        });
22603
22604        assert_eq!(
22605            worker.run_once().await.expect("cancelled attempt handled"),
22606            1
22607        );
22608        assert!(cancellation_observed.load(Ordering::SeqCst));
22609        assert_eq!(
22610            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22611            1
22612        );
22613
22614        let mut restarted = Worker::new(client, "rust-workers")
22615            .worker_id("rust-cancel-worker-restarted")
22616            .poll_timeout(Duration::from_millis(10));
22617        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22618        assert_eq!(
22619            restarted
22620                .run_once()
22621                .await
22622                .expect("replacement worker continues polling"),
22623            0
22624        );
22625    }
22626
22627    #[tokio::test]
22628    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22629        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22630        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22631        let client = Client::builder(server.base_url())
22632            .timeout(Duration::from_secs(2))
22633            .build()
22634            .expect("client");
22635
22636        let direct_error = client
22637            .complete_workflow_task(
22638                "workflow-timeout-task",
22639                "timeout-worker",
22640                3,
22641                vec![json!({
22642                    "type": "complete_workflow",
22643                    "result": fixture_envelope(Value::Null)
22644                })],
22645            )
22646            .await
22647            .expect_err("the low-level client preserves the completion rejection");
22648        let Error::Http { status, body } = direct_error else {
22649            panic!("expected the original HTTP completion rejection");
22650        };
22651        assert_eq!(status, reqwest::StatusCode::CONFLICT);
22652        assert_eq!(
22653            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22654            "run_timed_out"
22655        );
22656
22657        let mut worker = Worker::new(client, "rust-workers")
22658            .worker_id("timeout-worker")
22659            .poll_timeout(Duration::from_millis(10));
22660        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22661            Ok(json!({"late": "result"}))
22662        });
22663
22664        assert_eq!(
22665            worker
22666                .run_once()
22667                .await
22668                .expect("authoritative selected-run timeout settles the tick"),
22669            1
22670        );
22671        assert_eq!(
22672            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22673            2,
22674            "both the direct client proof and managed worker must see the rejection"
22675        );
22676    }
22677
22678    #[tokio::test]
22679    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22680        for (name, status, response) in [
22681            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22682            (
22683                "command was recorded",
22684                "409 Conflict",
22685                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22686            ),
22687            (
22688                "lease conflict",
22689                "409 Conflict",
22690                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22691            ),
22692            (
22693                "nonterminal run",
22694                "409 Conflict",
22695                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22696            ),
22697            (
22698                "different selected run",
22699                "409 Conflict",
22700                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22701            ),
22702            (
22703                "different task attempt",
22704                "409 Conflict",
22705                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22706            ),
22707            (
22708                "authentication failure",
22709                "401 Unauthorized",
22710                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22711            ),
22712            (
22713                "authorization failure",
22714                "403 Forbidden",
22715                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22716            ),
22717            (
22718                "protocol failure",
22719                "400 Bad Request",
22720                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22721            ),
22722            (
22723                "malformed command",
22724                "422 Unprocessable Entity",
22725                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22726            ),
22727            (
22728                "transient server failure",
22729                "503 Service Unavailable",
22730                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22731            ),
22732        ] {
22733            let server = MockWorkerServer::workflow_completion(status, response);
22734            let client = Client::builder(server.base_url())
22735                .timeout(Duration::from_secs(2))
22736                .build()
22737                .expect("client");
22738            let mut worker = Worker::new(client, "rust-workers")
22739                .worker_id("timeout-worker")
22740                .poll_timeout(Duration::from_millis(10));
22741            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22742                Ok(json!({"late": "result"}))
22743            });
22744
22745            let error = worker
22746                .run_once()
22747                .await
22748                .expect_err(&format!("{name} must remain an error"));
22749            assert!(
22750                matches!(error, Error::Http { .. } | Error::Protocol(_)),
22751                "{name} returned an unexpected error variant: {error}"
22752            );
22753        }
22754    }
22755
22756    #[tokio::test]
22757    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22758        let server = MockWorkerServer::start();
22759        let client = Client::builder(server.base_url())
22760            .worker_token(Some("worker-secret".to_string()))
22761            .namespace("orders")
22762            .timeout(Duration::from_secs(2))
22763            .build()
22764            .expect("client");
22765        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22766
22767        let result = client
22768            .deregister_worker_registration("worker/α space")
22769            .await
22770            .expect("deregister worker registration");
22771
22772        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22773        assert_eq!(
22774            server.worker_protocol_for(path).as_deref(),
22775            Some(WORKER_PROTOCOL_VERSION)
22776        );
22777        assert_eq!(server.control_protocol_for(path), None);
22778        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22779        assert_eq!(
22780            server.authorization_for(path).as_deref(),
22781            Some("Bearer worker-secret")
22782        );
22783        assert_eq!(
22784            result,
22785            WorkerDeregistrationEnvelope {
22786                worker_id: "deregistered-worker".to_string(),
22787                outcome: "deregistered".to_string(),
22788                recovered_workflow_task_count: 2,
22789            }
22790        );
22791    }
22792
22793    #[tokio::test]
22794    async fn low_level_registration_rejects_update_validators_before_transport() {
22795        let server = MockWorkerServer::start();
22796        let client = Client::builder(server.base_url())
22797            .timeout(Duration::from_secs(2))
22798            .build()
22799            .expect("client");
22800
22801        for update_validators in [json!(["approve"]), json!("approve")] {
22802            let error = client
22803                .register_worker_with_command_contracts(
22804                    "validator-claiming-worker",
22805                    "rust-workers",
22806                    vec!["orders".to_string()],
22807                    vec![],
22808                    1,
22809                    1,
22810                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22811                    json!({
22812                        "orders": {
22813                            "queries": ["current"],
22814                            "updates": ["approve"],
22815                            "update_validators": update_validators,
22816                        },
22817                    }),
22818                )
22819                .await
22820                .expect_err("unsupported validator claims must fail before registration");
22821
22822            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22823                panic!("expected typed unsupported-validator failure");
22824            };
22825            assert_eq!(workflow_type, "orders");
22826        }
22827        assert_eq!(server.request_count("/api/worker/register"), 0);
22828    }
22829
22830    #[tokio::test]
22831    async fn low_level_registration_preserves_query_and_update_contracts() {
22832        let server = MockWorkerServer::start();
22833        let client = Client::builder(server.base_url())
22834            .timeout(Duration::from_secs(2))
22835            .build()
22836            .expect("client");
22837        let contracts = json!({
22838            "orders": {
22839                "queries": ["current"],
22840                "updates": ["approve"],
22841                "update_validators": [],
22842            },
22843            "payments": {
22844                "queries": ["status"],
22845                "updates": ["capture"],
22846            },
22847        });
22848
22849        client
22850            .register_worker_with_command_contracts(
22851                "command-worker",
22852                "rust-workers",
22853                vec!["orders".to_string(), "payments".to_string()],
22854                vec![],
22855                1,
22856                1,
22857                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22858                contracts.clone(),
22859            )
22860            .await
22861            .expect("query and update contracts must remain supported");
22862
22863        assert_eq!(
22864            server.request_body("/api/worker/register")["workflow_command_contracts"],
22865            contracts
22866        );
22867    }
22868
22869    #[tokio::test]
22870    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22871        let server = MockWorkerServer::start();
22872        let control_only = Client::builder(server.base_url())
22873            .control_token(Some("control-secret".to_string()))
22874            .build()
22875            .expect("control client");
22876
22877        let error = control_only
22878            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22879            .await
22880            .expect_err("control token must not authorize a worker request");
22881        assert!(matches!(
22882            error,
22883            Error::MissingRoleCredentials { role: "worker", .. }
22884        ));
22885        assert_eq!(server.request_count("/api/worker/register"), 0);
22886
22887        let worker_only = Client::builder(server.base_url())
22888            .worker_token(Some("worker-secret".to_string()))
22889            .build()
22890            .expect("worker client");
22891        let error = worker_only
22892            .health()
22893            .await
22894            .expect_err("worker token must not authorize a control request");
22895        assert!(matches!(
22896            error,
22897            Error::MissingRoleCredentials {
22898                role: "control",
22899                ..
22900            }
22901        ));
22902        assert_eq!(server.request_count("/api/health"), 0);
22903    }
22904
22905    #[tokio::test]
22906    async fn shared_token_supports_worker_and_control_planes() {
22907        let server = MockWorkerServer::start();
22908        let client = Client::builder(server.base_url())
22909            .token(Some("shared-secret".to_string()))
22910            .build()
22911            .expect("client");
22912
22913        client.health().await.expect("control request");
22914        client
22915            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22916            .await
22917            .expect("worker request");
22918
22919        assert_eq!(
22920            server.authorization_for("/api/health").as_deref(),
22921            Some("Bearer shared-secret")
22922        );
22923        assert_eq!(
22924            server.control_protocol_for("/api/health").as_deref(),
22925            Some(CONTROL_PLANE_VERSION)
22926        );
22927        assert_eq!(
22928            server.authorization_for("/api/worker/register").as_deref(),
22929            Some("Bearer shared-secret")
22930        );
22931        assert_eq!(
22932            server
22933                .worker_protocol_for("/api/worker/register")
22934                .as_deref(),
22935            Some(WORKER_PROTOCOL_VERSION)
22936        );
22937    }
22938
22939    #[tokio::test]
22940    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22941        let server = MockWorkerServer::start();
22942        let client = Client::builder(server.base_url())
22943            .timeout(Duration::from_secs(2))
22944            .build()
22945            .expect("client");
22946
22947        client
22948            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22949            .await
22950            .expect("register");
22951        client
22952            .heartbeat_worker("capture-worker", 1, 1)
22953            .await
22954            .expect("heartbeat");
22955        client
22956            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22957            .await
22958            .expect("workflow poll");
22959        client
22960            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22961            .await
22962            .expect("activity poll");
22963
22964        for path in [
22965            "/api/worker/register",
22966            "/api/worker/heartbeat",
22967            "/api/worker/workflow-tasks/poll",
22968            "/api/worker/activity-tasks/poll",
22969        ] {
22970            assert_eq!(
22971                server.worker_protocol_for(path).as_deref(),
22972                Some(WORKER_PROTOCOL_VERSION),
22973                "unexpected protocol for {path}"
22974            );
22975        }
22976
22977        assert_eq!(
22978            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22979            1
22980        );
22981        assert_eq!(
22982            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22983            1
22984        );
22985        assert!(
22986            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22987                .as_str()
22988                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22989        );
22990        assert!(
22991            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22992                .as_str()
22993                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22994        );
22995    }
22996
22997    #[tokio::test]
22998    async fn query_task_endpoints_send_the_query_feature_protocol() {
22999        let server = MockWorkerServer::start();
23000        let client = Client::builder(server.base_url())
23001            .timeout(Duration::from_secs(2))
23002            .build()
23003            .expect("client");
23004
23005        client
23006            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23007            .await
23008            .expect("query poll");
23009        client
23010            .complete_query_task(
23011                "query-capture",
23012                "capture-worker",
23013                1,
23014                json!(8),
23015                DEFAULT_CODEC,
23016            )
23017            .await
23018            .expect("query complete");
23019        client
23020            .fail_query_task(
23021                "query-capture",
23022                "capture-worker",
23023                1,
23024                "failed",
23025                "query_rejected",
23026                "QueryFailed",
23027            )
23028            .await
23029            .expect("query fail");
23030
23031        for path in [
23032            "/api/worker/query-tasks/poll",
23033            "/api/worker/query-tasks/query-capture/complete",
23034            "/api/worker/query-tasks/query-capture/fail",
23035        ] {
23036            assert_eq!(
23037                server.worker_protocol_for(path).as_deref(),
23038                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
23039                "unexpected protocol for {path}"
23040            );
23041        }
23042
23043        assert_eq!(
23044            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
23045            1
23046        );
23047        assert!(
23048            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
23049                .as_str()
23050                .is_some_and(|id| id.starts_with("rust-query-poll-"))
23051        );
23052    }
23053
23054    #[tokio::test]
23055    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
23056        let server = MockWorkerServer::transient_worker_failures();
23057        let client = Client::builder(server.base_url())
23058            .timeout(Duration::from_secs(2))
23059            .build()
23060            .expect("client");
23061
23062        client
23063            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23064            .await
23065            .expect("workflow poll retry");
23066        client
23067            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23068            .await
23069            .expect("activity poll retry");
23070        client
23071            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23072            .await
23073            .expect("query poll retry");
23074
23075        for path in [
23076            "/api/worker/workflow-tasks/poll",
23077            "/api/worker/activity-tasks/poll",
23078            "/api/worker/query-tasks/poll",
23079        ] {
23080            let bodies = server.request_bodies(path);
23081            assert_eq!(bodies.len(), 2, "{path} must be retried once");
23082            assert_eq!(
23083                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
23084                "{path} must preserve the request binding across retry"
23085            );
23086        }
23087    }
23088
23089    #[tokio::test]
23090    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
23091        let server = MockWorkerServer::consecutive_poll_failures(2);
23092        let client = Client::builder(server.base_url())
23093            .timeout(Duration::from_secs(2))
23094            .build()
23095            .expect("client");
23096        let mut worker = Worker::new(client, "capture")
23097            .worker_id("capture-worker")
23098            .poll_timeout(Duration::from_millis(10))
23099            .retry_policy(WorkerRetryPolicy {
23100                max_retries: 2,
23101                initial_backoff: Duration::from_millis(1),
23102                max_backoff: Duration::from_millis(1),
23103            });
23104        worker.register_workflow(
23105            "capture.workflow",
23106            |_ctx, _input| async move { Ok(Value::Null) },
23107        );
23108        worker.register_activity(
23109            "capture.activity",
23110            |_ctx, _input| async move { Ok(Value::Null) },
23111        );
23112        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23113            Ok(Value::Null)
23114        });
23115
23116        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23117
23118        for path in [
23119            "/api/worker/workflow-tasks/poll",
23120            "/api/worker/activity-tasks/poll",
23121            "/api/worker/query-tasks/poll",
23122        ] {
23123            let bodies = server.request_bodies(path);
23124            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23125            assert!(
23126                bodies
23127                    .iter()
23128                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23129                "{path} must preserve one request binding across every retry"
23130            );
23131        }
23132    }
23133
23134    fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23135        let reason = if unavailable {
23136            "storage_admission_unavailable"
23137        } else {
23138            "storage_pressure"
23139        };
23140        let mut body = json!({
23141            "reason": reason,
23142            "storage_state": if unavailable { "fenced" } else { "draining" },
23143            "retryable": true,
23144            "retry_after_seconds": 1,
23145        });
23146        if !mid_poll {
23147            body["request_admitted"] = json!(false);
23148        }
23149        if let Some(id) = poll_id {
23150            body["task"] = Value::Null;
23151            body["poll_status"] = json!(reason);
23152            body["poll_request_id"] = json!(id);
23153            body["retry_same_poll_request_id"] = json!(true);
23154            body["claim_admitted"] = json!(false);
23155        }
23156        body
23157    }
23158
23159    fn backend_refusal(path: &str, request: &str) -> Option<Value> {
23160        let operation = match path {
23161            "/api/worker/register" => "register_worker",
23162            "/api/worker/heartbeat" => "heartbeat_worker",
23163            "/api/worker/workflow-tasks/poll" => "poll_workflow_task",
23164            "/api/worker/activity-tasks/poll" => "poll_activity_task",
23165            "/api/worker/query-tasks/poll" => "poll_query_task",
23166            "/api/worker/update-validation-tasks/poll" => "poll_update_validation_task",
23167            _ => return None,
23168        };
23169        let request: Value = serde_json::from_str(request).ok()?;
23170        let mut response = json!({
23171            "reason": "backend_unavailable",
23172            "operation": operation,
23173            "outcome": "unknown",
23174            "retryable": true,
23175            "retry_after_seconds": 1,
23176            "worker_id": request["worker_id"],
23177            "task_queue": request.get("task_queue"),
23178        });
23179        if path.ends_with("/poll") {
23180            response["task"] = Value::Null;
23181            response["poll_status"] = json!("backend_unavailable");
23182            response["poll_request_id"] = request["poll_request_id"].clone();
23183            response["retry_same_poll_request_id"] = json!(true);
23184        }
23185        Some(response)
23186    }
23187
23188    fn backend_retry_override(
23189        path: &str,
23190        request: &str,
23191        number: usize,
23192    ) -> Option<(&'static str, String)> {
23193        if number > 8 {
23194            return None;
23195        }
23196        Some((
23197            "503 Service Unavailable",
23198            backend_refusal(path, request)?.to_string(),
23199        ))
23200    }
23201
23202    fn backend_unavailable_override(
23203        path: &str,
23204        request: &str,
23205        _number: usize,
23206    ) -> Option<(&'static str, String)> {
23207        Some((
23208            "503 Service Unavailable",
23209            backend_refusal(path, request)?.to_string(),
23210        ))
23211    }
23212
23213    fn storage_worker(server: &MockWorkerServer) -> Worker {
23214        Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23215            .worker_id("storage-worker")
23216            .retry_policy(WorkerRetryPolicy {
23217                max_retries: 1,
23218                initial_backoff: Duration::from_millis(1),
23219                max_backoff: Duration::from_millis(1),
23220            })
23221    }
23222
23223    fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23224        let requests = server.requests.lock().expect("requests");
23225        let bodies: Vec<_> = requests
23226            .iter()
23227            .filter(|request| request.path == path)
23228            .map(|request| &request.body)
23229            .collect();
23230        assert_eq!(bodies.len(), count, "{path}");
23231        assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23232    }
23233
23234    #[test]
23235    fn backend_recovery_requires_the_explicit_worker_contract() {
23236        for (path, operation) in [
23237            ("/worker/register", "register_worker"),
23238            ("/worker/heartbeat", "heartbeat_worker"),
23239            ("/worker/workflow-tasks/poll", "poll_workflow_task"),
23240            ("/worker/activity-tasks/poll", "poll_activity_task"),
23241            ("/worker/query-tasks/poll", "poll_query_task"),
23242            (
23243                "/worker/update-validation-tasks/poll",
23244                "poll_update_validation_task",
23245            ),
23246        ] {
23247            let poll_id = path.ends_with("/poll").then_some("same-poll");
23248            let request = json!({"worker_id":"same-worker","task_queue":"same-queue","poll_request_id":poll_id});
23249            let body = backend_refusal(&format!("/api{path}"), &request.to_string())
23250                .expect("worker operation");
23251            assert_eq!(body["operation"], operation);
23252            let error = Error::Http {
23253                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23254                body: body.to_string(),
23255            };
23256            assert_eq!(
23257                worker_backend_unavailable_retry_after(&error, path, poll_id),
23258                Some(Duration::from_secs(1))
23259            );
23260            assert!(!worker_operation_is_retryable(&error));
23261            for (field, value) in [
23262                ("operation", json!("wrong_operation")),
23263                ("outcome", json!("completed")),
23264                ("retryable", json!(false)),
23265                ("retry_after_seconds", json!(0)),
23266                ("worker_id", json!("")),
23267            ] {
23268                let mut invalid = body.clone();
23269                invalid[field] = value;
23270                let error = Error::Http {
23271                    status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23272                    body: invalid.to_string(),
23273                };
23274                assert!(
23275                    worker_backend_unavailable_retry_after(&error, path, poll_id).is_none(),
23276                    "{path}: {field}"
23277                );
23278                assert!(!worker_operation_is_retryable(&error));
23279            }
23280            if let Some(poll_id) = poll_id {
23281                for (field, value) in [
23282                    ("poll_request_id", json!("wrong-poll")),
23283                    ("poll_status", json!("empty")),
23284                    ("task", json!({"task_id":"claimed"})),
23285                    ("retry_same_poll_request_id", json!(false)),
23286                ] {
23287                    let mut invalid = body.clone();
23288                    invalid[field] = value;
23289                    let error = Error::Http {
23290                        status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23291                        body: invalid.to_string(),
23292                    };
23293                    assert!(
23294                        worker_backend_unavailable_retry_after(&error, path, Some(poll_id))
23295                            .is_none(),
23296                        "{path}: {field}"
23297                    );
23298                }
23299            }
23300            let error = Error::Http {
23301                status: reqwest::StatusCode::UNAUTHORIZED,
23302                body: body.to_string(),
23303            };
23304            assert!(worker_backend_unavailable_retry_after(&error, path, poll_id).is_none());
23305            assert!(!worker_operation_is_retryable(&error));
23306        }
23307    }
23308
23309    #[tokio::test]
23310    async fn backend_recovery_outlives_generic_retry_budget_and_preserves_polls() {
23311        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23312            request_override: Some(backend_retry_override),
23313            ..MockWorkerBehavior::default()
23314        });
23315        let mut worker =
23316            storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23317        worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23318        worker.register_activity("backend.activity", |_, _| async { Ok(Value::Null) });
23319        worker.register_query("backend.workflow", "state", |_, _| async {
23320            Ok(Value::Null)
23321        });
23322        worker.register().await.expect("registration recovery");
23323        worker
23324            .client
23325            .heartbeat_worker("storage-worker", 1, 1)
23326            .await
23327            .expect("heartbeat recovery");
23328        assert_eq!(worker.run_once().await.expect("poll recovery"), 0);
23329        for path in [
23330            "/api/worker/register",
23331            "/api/worker/heartbeat",
23332            "/api/worker/workflow-tasks/poll",
23333            "/api/worker/activity-tasks/poll",
23334            "/api/worker/query-tasks/poll",
23335        ] {
23336            assert_identical_requests(&server, path, 9);
23337        }
23338    }
23339
23340    #[tokio::test]
23341    async fn backend_recovery_wait_stops_with_worker() {
23342        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23343            request_override: Some(backend_unavailable_override),
23344            ..MockWorkerBehavior::default()
23345        });
23346        let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy {
23347            max_retries: 1,
23348            initial_backoff: Duration::from_secs(2),
23349            max_backoff: Duration::from_secs(2),
23350        });
23351        worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23352        let started = tokio::time::Instant::now();
23353        let _ = tokio::time::timeout(
23354            Duration::from_secs(1),
23355            worker.run_until(tokio::time::sleep(Duration::from_millis(100))),
23356        )
23357        .await
23358        .expect("shutdown interrupts backend wait");
23359        assert!(started.elapsed() < Duration::from_secs(1));
23360        assert!(server.request_count("/api/worker/register") >= 1);
23361    }
23362
23363    #[test]
23364    fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23365        for unavailable in [false, true] {
23366            for mid_poll in [false, true] {
23367                let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23368                let error = Error::Http {
23369                    status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23370                    body: body.to_string(),
23371                };
23372                assert_eq!(
23373                    worker_storage_admission_retry_after(&error, Some("same-poll")),
23374                    Some(Duration::from_secs(1))
23375                );
23376                assert!(
23377                    !worker_operation_is_retryable(&error),
23378                    "storage is not a bounded generic retry"
23379                );
23380                for (field, value) in [
23381                    ("poll_request_id", json!("wrong-poll")),
23382                    ("task", json!({"task_id":"claimed"})),
23383                    ("retryable", json!(false)),
23384                    ("retry_after_seconds", json!(0)),
23385                    ("retry_after_seconds", json!(true)),
23386                    ("retry_after_seconds", json!(1.0)),
23387                    ("storage_state", json!("normal")),
23388                    ("poll_status", json!("empty")),
23389                    ("claim_admitted", json!(true)),
23390                    ("retry_same_poll_request_id", json!(false)),
23391                    ("request_admitted", json!(true)),
23392                ] {
23393                    let mut invalid = body.clone();
23394                    invalid[field] = value;
23395                    let error = Error::Http {
23396                        status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23397                        body: invalid.to_string(),
23398                    };
23399                    assert!(
23400                        worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23401                        "{field}"
23402                    );
23403                }
23404            }
23405        }
23406        let body = storage_refusal(None, false, false);
23407        let error = Error::Http {
23408            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23409            body: body.to_string(),
23410        };
23411        assert!(worker_storage_admission_retry_after(&error, None).is_some());
23412        assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23413        let error = Error::Http {
23414            status: reqwest::StatusCode::FORBIDDEN,
23415            body: body.to_string(),
23416        };
23417        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23418        let body = storage_refusal(None, false, true);
23419        let error = Error::Http {
23420            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23421            body: body.to_string(),
23422        };
23423        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23424    }
23425
23426    #[tokio::test]
23427    async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23428        for unavailable in [false, true] {
23429            for mid_poll in [false, true] {
23430                let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23431                    poll_failures_per_path: 1,
23432                    storage_refusals: 7,
23433                    storage_path: Some("/poll"),
23434                    storage_unavailable: unavailable,
23435                    storage_mid_poll: mid_poll,
23436                    ..MockWorkerBehavior::default()
23437                });
23438                let mut worker = storage_worker(&server);
23439                worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23440                assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23441                for path in [
23442                    "/api/worker/workflow-tasks/poll",
23443                    "/api/worker/activity-tasks/poll",
23444                    "/api/worker/query-tasks/poll",
23445                ] {
23446                    assert_identical_requests(&server, path, 9);
23447                }
23448            }
23449        }
23450    }
23451
23452    #[tokio::test]
23453    async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23454        struct CountedBody(Arc<AtomicUsize>);
23455        impl Serialize for CountedBody {
23456            fn serialize<S: Serializer>(
23457                &self,
23458                serializer: S,
23459            ) -> std::result::Result<S::Ok, S::Error> {
23460                let count = self.0.fetch_add(1, Ordering::SeqCst);
23461                json!({"serialization":count,"lease_owner":"worker","attempt":7})
23462                    .serialize(serializer)
23463            }
23464        }
23465        for path in [
23466            "/api/worker/register",
23467            "/api/worker/heartbeat",
23468            "/api/worker/workflow-tasks/storage-task/complete",
23469            "/api/worker/workflow-tasks/storage-task/fail",
23470            "/api/worker/activity-tasks/storage-task/complete",
23471            "/api/worker/activity-tasks/storage-task/fail",
23472            "/api/worker/activity-tasks/storage-task/heartbeat",
23473            "/api/worker/query-tasks/storage-task/complete",
23474            "/api/worker/query-tasks/storage-task/fail",
23475        ] {
23476            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23477                storage_refusals: 7,
23478                storage_path: Some(path),
23479                ..MockWorkerBehavior::default()
23480            });
23481            let worker =
23482                storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23483            let calls = Arc::new(AtomicUsize::new(0));
23484            let _: Value = worker
23485                .client
23486                .request_json(
23487                    reqwest::Method::POST,
23488                    &path[4..],
23489                    RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23490                    Some(&CountedBody(Arc::clone(&calls))),
23491                )
23492                .await
23493                .expect("prepared request recovery");
23494            assert_eq!(calls.load(Ordering::SeqCst), 1);
23495            assert_identical_requests(&server, path, 8);
23496        }
23497        for worker_scope in [false, true] {
23498            let path = if worker_scope {
23499                "/api/health"
23500            } else {
23501                "/api/worker/register"
23502            };
23503            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23504                storage_refusals: usize::MAX,
23505                storage_path: Some(path),
23506                ..MockWorkerBehavior::default()
23507            });
23508            let worker = storage_worker(&server);
23509            let client = worker.client.clone();
23510            let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
23511            let error = if worker_scope {
23512                worker
23513                    .client
23514                    .health()
23515                    .await
23516                    .expect_err("control plane is not retried")
23517            } else {
23518                client
23519                    .request_json::<Value, Value>(
23520                        reqwest::Method::POST,
23521                        "/worker/register",
23522                        RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23523                        Some(&json!({})),
23524                    )
23525                    .await
23526                    .expect_err("direct client is not retried")
23527            };
23528            assert!(worker_storage_admission_body(&error).is_some());
23529            assert_eq!(server.request_count(path), 1);
23530        }
23531    }
23532
23533    #[tokio::test]
23534    async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
23535        for fail in [false, true] {
23536            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23537                storage_activity: true,
23538                storage_refusals: 7,
23539                storage_path: Some("/storage-activity/"),
23540                ..MockWorkerBehavior::default()
23541            });
23542            let mut worker = storage_worker(&server);
23543            let calls = Arc::new(AtomicUsize::new(0));
23544            let observed = Arc::clone(&calls);
23545            worker.register_activity("storage.activity", move |ctx, _| {
23546                let calls = Arc::clone(&observed);
23547                async move {
23548                    calls.fetch_add(1, Ordering::SeqCst);
23549                    ctx.heartbeat(json!({"step":2})).await?;
23550                    if fail {
23551                        Err(Error::WorkerLoop("intentional handler failure".to_string()))
23552                    } else {
23553                        Ok(json!({"receipt":true}))
23554                    }
23555                }
23556            });
23557            assert_eq!(worker.run_once().await.expect("activity settled"), 1);
23558            assert_eq!(calls.load(Ordering::SeqCst), 1);
23559            assert_identical_requests(
23560                &server,
23561                "/api/worker/activity-tasks/storage-activity/heartbeat",
23562                8,
23563            );
23564            let suffix = if fail { "fail" } else { "complete" };
23565            assert_identical_requests(
23566                &server,
23567                &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
23568                8,
23569            );
23570            let other = if fail { "complete" } else { "fail" };
23571            assert_eq!(
23572                server.request_count(&format!(
23573                    "/api/worker/activity-tasks/storage-activity/{other}"
23574                )),
23575                0
23576            );
23577        }
23578    }
23579
23580    #[tokio::test]
23581    async fn storage_waits_are_interruptible_without_false_activity_failure() {
23582        for path in [
23583            "/api/worker/register",
23584            "/api/worker/heartbeat",
23585            "/api/worker/activity-tasks/poll",
23586            "/api/worker/activity-tasks/storage-activity/heartbeat",
23587            "/api/worker/activity-tasks/storage-activity/complete",
23588        ] {
23589            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23590                storage_activity: true,
23591                storage_refusals: usize::MAX,
23592                storage_path: Some(path),
23593                ..MockWorkerBehavior::default()
23594            });
23595            let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23596            worker.register_activity("storage.activity", |ctx, _| async move {
23597                ctx.heartbeat(json!({"step":2})).await?;
23598                Ok(json!({"receipt":true}))
23599            });
23600            let shutdown = async {
23601                while server.request_count(path) == 0 {
23602                    tokio::time::sleep(Duration::from_millis(1)).await;
23603                }
23604            };
23605            let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
23606                .await
23607                .expect("shutdown interrupts admission");
23608            assert!(
23609                result.is_err(),
23610                "a refused operation must not appear acknowledged: {path}, {result:?}"
23611            );
23612            assert_eq!(server.request_count(path), 1);
23613            assert_eq!(
23614                server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
23615                0
23616            );
23617            assert_eq!(
23618                server.request_count("/api/worker/registrations/mock-worker"),
23619                usize::from(!path.ends_with("/register"))
23620            );
23621        }
23622    }
23623
23624    #[tokio::test]
23625    async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
23626        for fail in [false, true] {
23627            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23628                storage_query: true,
23629                storage_refusals: 7,
23630                storage_path: Some("/storage-query/"),
23631                ..MockWorkerBehavior::default()
23632            });
23633            let mut worker = storage_worker(&server);
23634            worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
23635            let calls = Arc::new(AtomicUsize::new(0));
23636            let observed = Arc::clone(&calls);
23637            worker.register_query("storage.workflow", "state", move |_, _| {
23638                let calls = Arc::clone(&observed);
23639                async move {
23640                    calls.fetch_add(1, Ordering::SeqCst);
23641                    if fail {
23642                        Err(Error::WorkerLoop("intentional query failure".to_string()))
23643                    } else {
23644                        Ok(json!({"state":"waiting"}))
23645                    }
23646                }
23647            });
23648            assert_eq!(worker.run_once().await.expect("query settled"), 1);
23649            assert_eq!(calls.load(Ordering::SeqCst), 1);
23650            let suffix = if fail { "fail" } else { "complete" };
23651            assert_identical_requests(
23652                &server,
23653                &format!("/api/worker/query-tasks/storage-query/{suffix}"),
23654                8,
23655            );
23656            let other = if fail { "complete" } else { "fail" };
23657            assert_eq!(
23658                server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
23659                0
23660            );
23661        }
23662    }
23663
23664    #[tokio::test]
23665    async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
23666        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23667            storage_refusals: 7,
23668            storage_path: Some("/poll"),
23669            unauthorized_polls: true,
23670            ..MockWorkerBehavior::default()
23671        });
23672        let error = storage_worker(&server)
23673            .run_once()
23674            .await
23675            .expect_err("auth remains terminal");
23676        assert!(matches!(
23677            error,
23678            Error::Http {
23679                status: reqwest::StatusCode::UNAUTHORIZED,
23680                ..
23681            }
23682        ));
23683        assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
23684
23685        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23686            storage_refusals: 7,
23687            storage_path: Some("/activity-cancel/complete"),
23688            ..MockWorkerBehavior::default()
23689        });
23690        let worker =
23691            storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23692        let error = worker
23693            .client
23694            .complete_activity_task(
23695                "activity-cancel",
23696                "attempt-cancel",
23697                "worker",
23698                json!({}),
23699                DEFAULT_CODEC,
23700            )
23701            .await
23702            .expect_err("cancellation remains terminal");
23703        assert!(activity_task_rejection_is_final(&error));
23704        assert_identical_requests(
23705            &server,
23706            "/api/worker/activity-tasks/activity-cancel/complete",
23707            8,
23708        );
23709
23710        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23711            storage_refusals: usize::MAX,
23712            storage_path: Some("/poll"),
23713            storage_wrong_poll_id: true,
23714            ..MockWorkerBehavior::default()
23715        });
23716        assert!(storage_worker(&server).run_once().await.is_err());
23717        assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
23718    }
23719
23720    #[tokio::test]
23721    async fn storage_pollers_stop_when_the_run_future_is_aborted() {
23722        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23723            storage_refusals: usize::MAX,
23724            storage_path: Some("/poll"),
23725            ..MockWorkerBehavior::default()
23726        });
23727        let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23728        worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
23729        let run = tokio::spawn(async move { worker.run().await });
23730        tokio::time::timeout(Duration::from_secs(2), async {
23731            while server.request_count("/api/worker/activity-tasks/poll") == 0 {
23732                tokio::time::sleep(Duration::from_millis(1)).await;
23733            }
23734        })
23735        .await
23736        .expect("poll started");
23737        run.abort();
23738        assert!(run.await.expect_err("cancelled run").is_cancelled());
23739        tokio::time::sleep(Duration::from_millis(250)).await;
23740        assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
23741    }
23742
23743    #[tokio::test]
23744    async fn query_protocol_rejection_from_older_server_is_typed() {
23745        let server = MockWorkerServer::reject_query_protocol();
23746        let client = Client::builder(server.base_url())
23747            .timeout(Duration::from_secs(2))
23748            .build()
23749            .expect("client");
23750
23751        let error = client
23752            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23753            .await
23754            .expect_err("server below query protocol floor must reject");
23755        let Error::Protocol(failure) = error else {
23756            panic!("expected typed protocol failure");
23757        };
23758
23759        assert_eq!(failure.status, 400);
23760        assert_eq!(failure.reason, "unsupported_protocol_version");
23761        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
23762        assert_eq!(
23763            failure.requested_version.as_deref(),
23764            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23765        );
23766        assert_eq!(
23767            server
23768                .worker_protocol_for("/api/worker/query-tasks/poll")
23769                .as_deref(),
23770            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23771        );
23772    }
23773
23774    #[tokio::test]
23775    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
23776        let server = MockWorkerServer::reject_query_protocol();
23777        let client = Client::builder(server.base_url())
23778            .timeout(Duration::from_secs(2))
23779            .build()
23780            .expect("client");
23781        let mut worker = Worker::new(client, "rust-workers")
23782            .worker_id("baseline-worker")
23783            .poll_timeout(Duration::from_millis(10));
23784
23785        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
23786            Ok(Value::Null)
23787        });
23788
23789        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
23790        assert_eq!(
23791            server
23792                .worker_protocol_for("/api/worker/workflow-tasks/poll")
23793                .as_deref(),
23794            Some(WORKER_PROTOCOL_VERSION)
23795        );
23796        assert_eq!(
23797            server.worker_protocol_for("/api/worker/query-tasks/poll"),
23798            None,
23799            "a worker without query handlers must not use the query-task endpoint"
23800        );
23801    }
23802
23803    #[tokio::test]
23804    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
23805        let server = MockWorkerServer::reject_query_completion();
23806        let client = Client::builder(server.base_url())
23807            .timeout(Duration::from_secs(2))
23808            .build()
23809            .expect("client");
23810
23811        let error = client
23812            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
23813            .await
23814            .expect_err("expired completion must be rejected");
23815        let Error::QueryFailed(failure) = error else {
23816            panic!("expected typed query failure");
23817        };
23818        assert_eq!(failure.status, 409);
23819        assert_eq!(failure.reason, "query_task_timed_out");
23820
23821        let mut worker = Worker::new(client, "rust-workers")
23822            .worker_id("late-worker")
23823            .poll_timeout(Duration::from_millis(10));
23824        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23825        worker.register_query(
23826            "counter",
23827            "current",
23828            |_ctx, _args| async move { Ok(json!(8)) },
23829        );
23830
23831        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
23832        assert_eq!(
23833            worker
23834                .run_once()
23835                .await
23836                .expect("worker continues after late completion"),
23837            0
23838        );
23839        assert_eq!(
23840            server.request_count("/api/worker/query-tasks/query-late/complete"),
23841            2
23842        );
23843        assert_eq!(
23844            server.request_count("/api/worker/query-tasks/query-late/fail"),
23845            0,
23846            "a server completion rejection must not be reported as an encoding failure"
23847        );
23848    }
23849
23850    #[tokio::test]
23851    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
23852        let server = MockWorkerServer::start();
23853        let client = Client::builder(server.base_url())
23854            .timeout(Duration::from_secs(2))
23855            .build()
23856            .expect("client");
23857        let mut worker = Worker::new(client, "rust-workers")
23858            .worker_id("joined-worker")
23859            .poll_timeout(Duration::from_millis(10));
23860        worker.register_workflow(
23861            "joined.workflow",
23862            |_ctx, _input| async move { Ok(Value::Null) },
23863        );
23864        worker.register_activity(
23865            "joined.activity",
23866            |_ctx, _input| async move { Ok(Value::Null) },
23867        );
23868        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
23869            Ok(Value::Null)
23870        });
23871
23872        worker
23873            .run_until(tokio::time::sleep(Duration::from_millis(20)))
23874            .await
23875            .expect("normal shutdown");
23876
23877        let deregistration_path = "/api/worker/registrations/mock-worker";
23878        assert_eq!(server.request_count(deregistration_path), 1);
23879        for poll_path in [
23880            "/api/worker/workflow-tasks/poll",
23881            "/api/worker/activity-tasks/poll",
23882            "/api/worker/query-tasks/poll",
23883        ] {
23884            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
23885        }
23886        assert_eq!(
23887            server.captured_paths().last().map(String::as_str),
23888            Some(deregistration_path),
23889            "deregistration must start only after every poller has joined"
23890        );
23891    }
23892
23893    #[tokio::test]
23894    async fn registration_failure_does_not_deregister() {
23895        let server = MockWorkerServer::rejected_registration();
23896        let client = Client::builder(server.base_url())
23897            .timeout(Duration::from_secs(2))
23898            .build()
23899            .expect("client");
23900        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
23901
23902        let error = worker
23903            .run_until(async {})
23904            .await
23905            .expect_err("registration must fail");
23906        assert!(matches!(
23907            error,
23908            Error::Http {
23909                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23910                ..
23911            }
23912        ));
23913        assert!(server
23914            .captured_paths()
23915            .iter()
23916            .all(|path| !path.starts_with("/api/worker/registrations/")));
23917    }
23918
23919    #[tokio::test]
23920    async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
23921        let server = MockWorkerServer::rejected_registration_protocol();
23922        let client = Client::builder(server.base_url())
23923            .timeout(Duration::from_secs(2))
23924            .build()
23925            .expect("client");
23926        let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
23927
23928        let error = worker
23929            .run_until(async {})
23930            .await
23931            .expect_err("a protocol 1.16 server must reject this worker");
23932        let Error::Protocol(failure) = error else {
23933            panic!("expected typed protocol rejection");
23934        };
23935        assert_eq!(failure.reason, "unsupported_protocol_version");
23936        assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
23937        assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
23938        assert_eq!(
23939            server
23940                .worker_protocol_for("/api/worker/register")
23941                .as_deref(),
23942            Some(WORKER_PROTOCOL_VERSION)
23943        );
23944    }
23945
23946    #[tokio::test]
23947    async fn declined_registration_does_not_deregister() {
23948        let server = MockWorkerServer::declined_registration();
23949        let client = Client::builder(server.base_url())
23950            .timeout(Duration::from_secs(2))
23951            .build()
23952            .expect("client");
23953        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
23954
23955        let error = worker
23956            .run_until(async {})
23957            .await
23958            .expect_err("declined registration must fail");
23959        assert!(matches!(error, Error::WorkerLoop(_)));
23960        assert!(error.to_string().contains("was not accepted"));
23961        assert!(server
23962            .captured_paths()
23963            .iter()
23964            .all(|path| !path.starts_with("/api/worker/registrations/")));
23965    }
23966
23967    #[tokio::test]
23968    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23969        let server = MockWorkerServer::rejected_deregistration();
23970        let client = Client::builder(server.base_url())
23971            .timeout(Duration::from_secs(2))
23972            .build()
23973            .expect("client");
23974        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23975
23976        let error = worker
23977            .run_until(async {})
23978            .await
23979            .expect_err("deregistration must fail");
23980        assert!(matches!(
23981            error,
23982            Error::Http {
23983                status: reqwest::StatusCode::FORBIDDEN,
23984                ..
23985            }
23986        ));
23987        assert_eq!(
23988            server.request_count("/api/worker/registrations/mock-worker"),
23989            1
23990        );
23991    }
23992
23993    #[tokio::test]
23994    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23995        let server = MockWorkerServer::rejected_deregistration_protocol();
23996        let client = Client::builder(server.base_url())
23997            .timeout(Duration::from_secs(2))
23998            .build()
23999            .expect("client");
24000        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
24001
24002        let error = worker
24003            .run_until(async {})
24004            .await
24005            .expect_err("protocol rejection must fail shutdown");
24006        let Error::Protocol(failure) = error else {
24007            panic!("expected typed protocol failure");
24008        };
24009        assert_eq!(failure.reason, "unsupported_protocol_version");
24010        assert_eq!(
24011            failure.requested_version.as_deref(),
24012            Some(WORKER_PROTOCOL_VERSION)
24013        );
24014        assert_eq!(
24015            server.request_count("/api/worker/registrations/mock-worker"),
24016            1
24017        );
24018    }
24019
24020    #[tokio::test]
24021    async fn primary_poller_error_retains_deregistration_failure_context() {
24022        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
24023        let client = Client::builder(server.base_url())
24024            .timeout(Duration::from_secs(2))
24025            .build()
24026            .expect("client");
24027        let mut worker = Worker::new(client, "rust-workers")
24028            .worker_id("combined-failure")
24029            .poll_timeout(Duration::from_millis(10));
24030        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
24031            Ok(Value::Null)
24032        });
24033
24034        let error = worker
24035            .run()
24036            .await
24037            .expect_err("worker and cleanup must fail");
24038        let summary = error.to_string();
24039        assert!(summary.contains("authentication_failed"));
24040        assert!(summary.contains("worker cannot deregister"));
24041        let Error::WorkerShutdown {
24042            primary,
24043            deregistration,
24044        } = error
24045        else {
24046            panic!("expected combined worker shutdown error");
24047        };
24048        assert!(matches!(
24049            *primary,
24050            Error::Http {
24051                status: reqwest::StatusCode::UNAUTHORIZED,
24052                ..
24053            }
24054        ));
24055        assert!(matches!(
24056            *deregistration,
24057            Error::Http {
24058                status: reqwest::StatusCode::FORBIDDEN,
24059                ..
24060            }
24061        ));
24062        assert_eq!(
24063            server.request_count("/api/worker/registrations/mock-worker"),
24064            1
24065        );
24066    }
24067
24068    #[tokio::test]
24069    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
24070        let server = MockWorkerServer::start();
24071        let client = Client::builder(server.base_url())
24072            .timeout(Duration::from_secs(2))
24073            .build()
24074            .expect("client");
24075        let mut worker = Worker::new(client, "rust-workers")
24076            .worker_id("activity-only-worker")
24077            .poll_timeout(Duration::from_millis(10));
24078
24079        worker.register_activity(
24080            "activity.only",
24081            |_ctx, _args| async move { Ok(Value::Null) },
24082        );
24083
24084        worker.run_until(async {}).await.expect("run worker");
24085    }
24086
24087    #[tokio::test]
24088    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
24089        let server = MockWorkerServer::start();
24090        let client = Client::builder(server.base_url())
24091            .timeout(Duration::from_secs(2))
24092            .build()
24093            .expect("client");
24094        let mut worker = Worker::new(client, "rust-workers")
24095            .worker_id("workflow-only-worker")
24096            .poll_timeout(Duration::from_millis(10));
24097
24098        worker.register_workflow(
24099            "workflow.only",
24100            |_ctx, _input| async move { Ok(Value::Null) },
24101        );
24102
24103        worker.run_until(async {}).await.expect("run worker");
24104    }
24105
24106    #[tokio::test]
24107    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
24108        let server = MockWorkerServer::start();
24109        let client = Client::builder(server.base_url())
24110            .timeout(Duration::from_secs(2))
24111            .build()
24112            .expect("client");
24113        let observations = Arc::new(Mutex::new(Vec::new()));
24114        let observed = Arc::clone(&observations);
24115        let mut worker = Worker::new(client, "rust-workers")
24116            .worker_id("observed-heartbeat-worker")
24117            .poll_timeout(Duration::from_millis(10))
24118            .on_worker_heartbeat(move |observation| {
24119                observed
24120                    .lock()
24121                    .expect("heartbeat observations")
24122                    .push(observation.clone());
24123            });
24124
24125        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
24126            Ok(Value::Null)
24127        });
24128        let acknowledged = Arc::clone(&observations);
24129        worker
24130            .run_until(async move {
24131                tokio::time::timeout(Duration::from_secs(2), async move {
24132                    loop {
24133                        if !acknowledged
24134                            .lock()
24135                            .expect("heartbeat observations")
24136                            .is_empty()
24137                        {
24138                            break;
24139                        }
24140                        tokio::time::sleep(Duration::from_millis(1)).await;
24141                    }
24142                })
24143                .await
24144                .expect("heartbeat acknowledgement within timeout");
24145            })
24146            .await
24147            .expect("run worker");
24148
24149        let observations = observations.lock().expect("heartbeat observations");
24150        let first = observations.first().expect("heartbeat acknowledgement");
24151        assert_eq!(first.worker_id, "observed-heartbeat-worker");
24152        assert_eq!(first.task_queue, "rust-workers");
24153        assert!(first.acknowledged_at_unix_millis > 0);
24154        assert_eq!(first.acknowledgement, json!({}));
24155    }
24156
24157    #[tokio::test]
24158    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
24159        let server = MockWorkerServer::delayed_heartbeat_worker();
24160        let client = Client::builder(server.base_url())
24161            .timeout(Duration::from_secs(3))
24162            .build()
24163            .expect("client");
24164        let observations = Arc::new(Mutex::new(Vec::new()));
24165        let observed = Arc::clone(&observations);
24166        let mut worker = Worker::new(client, "rust-snapshot-workers")
24167            .worker_id("rust-snapshot-worker")
24168            .poll_timeout(Duration::from_millis(10))
24169            .on_worker_heartbeat(move |observation| {
24170                observed
24171                    .lock()
24172                    .expect("heartbeat observations")
24173                    .push(observation.clone());
24174            });
24175
24176        worker.register_workflow("snapshot", |ctx, _input| async move {
24177            ctx.wait_signal("finish").await?;
24178            Ok(json!({"status": "finished"}))
24179        });
24180        worker.register_query("snapshot", "current", |ctx, _args| async move {
24181            Ok(json!(ctx
24182                .signals("increment")
24183                .iter()
24184                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24185                .sum::<i64>()))
24186        });
24187        worker.register_activity("cancel-aware", |_ctx, _args| async move {
24188            Ok(json!({"late": "completion"}))
24189        });
24190
24191        worker
24192            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
24193            .await
24194            .expect("delayed heartbeat must allow a clean worker shutdown");
24195
24196        let observations = observations.lock().expect("heartbeat observations");
24197        assert!(
24198            observations.len() >= 3,
24199            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
24200        );
24201        assert!(
24202            observations.windows(2).all(|pair| {
24203                pair[1].acknowledged_at_unix_millis
24204                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
24205                    >= 850
24206            }),
24207            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
24208        );
24209        drop(observations);
24210
24211        let heartbeat_times = server.request_times("/api/worker/heartbeat");
24212        let delayed_request_at = *heartbeat_times
24213            .get(1)
24214            .expect("intentionally delayed heartbeat request");
24215        let delay_window_start = delayed_request_at + Duration::from_millis(100);
24216        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
24217        for path in [
24218            "/api/worker/workflow-tasks/poll",
24219            "/api/worker/activity-tasks/poll",
24220            "/api/worker/query-tasks/poll",
24221        ] {
24222            assert!(
24223                server
24224                    .request_times(path)
24225                    .iter()
24226                    .any(|received_at| *received_at >= delay_window_start
24227                        && *received_at <= delay_window_end),
24228                "{path} must keep polling while a heartbeat acknowledgement is delayed"
24229            );
24230        }
24231        assert!(
24232            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
24233            "workflow work must be settled"
24234        );
24235        assert!(
24236            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
24237            "activity work must be settled"
24238        );
24239        assert!(
24240            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
24241            "query work must be settled"
24242        );
24243    }
24244
24245    #[tokio::test]
24246    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
24247        let server = MockWorkerServer::heartbeat_retry_worker();
24248        let client = Client::builder(server.base_url())
24249            .timeout(Duration::from_secs(2))
24250            .build()
24251            .expect("client");
24252        let observations = Arc::new(Mutex::new(Vec::new()));
24253        let observed = Arc::clone(&observations);
24254        let worker = Worker::new(client, "rust-workers")
24255            .worker_id("heartbeat-retry-worker")
24256            .retry_policy(WorkerRetryPolicy {
24257                max_retries: 1,
24258                initial_backoff: Duration::from_millis(300),
24259                max_backoff: Duration::from_millis(300),
24260            })
24261            .on_worker_heartbeat(move |observation| {
24262                observed
24263                    .lock()
24264                    .expect("heartbeat observations")
24265                    .push(observation.clone());
24266            });
24267
24268        worker
24269            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
24270            .await
24271            .expect("retryable heartbeat failure must remain bounded and recover");
24272
24273        let observations = observations.lock().expect("heartbeat observations");
24274        assert!(observations.len() >= 3, "heartbeat retry must recover");
24275        assert!(
24276            observations.windows(2).all(|pair| {
24277                pair[1]
24278                    .acknowledged_at_unix_millis
24279                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
24280                    >= 850
24281            }),
24282            "a successful retry must start a fresh advertised cadence: {observations:?}"
24283        );
24284        assert_eq!(
24285            server.request_count("/api/worker/heartbeat"),
24286            observations.len() + 1,
24287            "one retryable failure must add exactly one bounded request"
24288        );
24289    }
24290
24291    #[tokio::test]
24292    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
24293        let server = MockWorkerServer::waiting_query_worker();
24294        let client = Client::builder(server.base_url())
24295            .timeout(Duration::from_secs(2))
24296            .build()
24297            .expect("client");
24298        let observations = Arc::new(Mutex::new(Vec::new()));
24299        let observed = Arc::clone(&observations);
24300        let mut worker = Worker::new(client, "rust-snapshot-workers")
24301            .worker_id("rust-snapshot-worker")
24302            .poll_timeout(Duration::from_millis(10))
24303            .on_worker_heartbeat(move |observation| {
24304                observed
24305                    .lock()
24306                    .expect("heartbeat observations")
24307                    .push(observation.clone());
24308            });
24309
24310        worker.register_workflow("snapshot", |ctx, _input| async move {
24311            ctx.wait_signal("finish").await?;
24312            Ok(json!({"status": "finished"}))
24313        });
24314        worker.register_query("snapshot", "current", |ctx, _args| async move {
24315            let current = ctx
24316                .signals("increment")
24317                .iter()
24318                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24319                .sum::<i64>();
24320            Ok(json!(current))
24321        });
24322        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24323
24324        worker
24325            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24326            .await
24327            .expect("pending workflow and query poller must remain live until shutdown");
24328
24329        assert!(
24330            observations.lock().expect("heartbeat observations").len() >= 4,
24331            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24332        );
24333        assert!(
24334            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24335            "workflow polling must continue after empty replay acknowledgements"
24336        );
24337        assert!(
24338            server.request_count("/api/worker/query-tasks/poll") >= 2,
24339            "query polling must continue after serving the current query"
24340        );
24341        assert_eq!(
24342            server.request_body("/api/worker/register")["capabilities"],
24343            json!([
24344                CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24345                DURABLE_SELECTION_CAPABILITY,
24346                MEMO_UPSERTS_CAPABILITY,
24347                TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24348                QUERY_TASKS_CAPABILITY,
24349                WORKFLOW_UPDATES_CAPABILITY,
24350                MESSAGE_STREAMS_CAPABILITY
24351            ])
24352        );
24353        assert_eq!(
24354            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24355            json!({
24356                "queries": ["current"],
24357                "query_contracts": [],
24358                "signals": [],
24359                "signal_contracts": [],
24360                "updates": ["replace"],
24361                "update_contracts": [],
24362                "update_validators": [],
24363            })
24364        );
24365
24366        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24367        assert_eq!(
24368            opened["commands"],
24369            json!([{
24370                "type": "open_signal_wait",
24371                "signal_name": "finish",
24372            }])
24373        );
24374
24375        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24376            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24377            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24378            let failure = server.request_body(&fail_path);
24379            assert_eq!(
24380                failure["failure"]["type"],
24381                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24382            );
24383            assert_eq!(server.request_count(&completion_path), 0);
24384        }
24385
24386        let query_completion =
24387            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24388        assert_eq!(query_completion["result"], json!(8));
24389
24390        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24391        assert_eq!(
24392            server.request_count(terminal_path),
24393            1,
24394            "the matching signal must settle the workflow exactly once"
24395        );
24396        let terminal = server.request_body(terminal_path);
24397        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24398        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24399        assert_eq!(
24400            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24401                .expect("terminal workflow result"),
24402            json!({"status": "finished"})
24403        );
24404    }
24405
24406    #[tokio::test]
24407    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24408        let server = MockWorkerServer::transient_worker_failures();
24409        let client = Client::builder(server.base_url())
24410            .timeout(Duration::from_secs(2))
24411            .build()
24412            .expect("client");
24413        let mut worker = Worker::new(client, "rust-workers")
24414            .worker_id("retry-worker")
24415            .poll_timeout(Duration::from_millis(10))
24416            .retry_policy(WorkerRetryPolicy {
24417                max_retries: 2,
24418                initial_backoff: Duration::from_millis(1),
24419                max_backoff: Duration::from_millis(1),
24420            });
24421        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24422        worker.register_activity(
24423            "counter.activity",
24424            |_ctx, _input| async move { Ok(Value::Null) },
24425        );
24426        worker.register_query(
24427            "counter",
24428            "current",
24429            |_ctx, _args| async move { Ok(json!(8)) },
24430        );
24431
24432        worker
24433            .run_until(tokio::time::sleep(Duration::from_millis(75)))
24434            .await
24435            .expect("transient failures must not stop the worker");
24436
24437        for path in [
24438            "/api/worker/heartbeat",
24439            "/api/worker/workflow-tasks/poll",
24440            "/api/worker/activity-tasks/poll",
24441            "/api/worker/query-tasks/poll",
24442        ] {
24443            assert!(
24444                server.request_count(path) >= 2,
24445                "{path} must continue after its transient failure"
24446            );
24447        }
24448    }
24449
24450    #[tokio::test]
24451    async fn worker_continues_after_long_poll_capacity_backpressure() {
24452        let server = MockWorkerServer::capacity_limited_activity_poll();
24453        let client = Client::builder(server.base_url())
24454            .timeout(Duration::from_secs(2))
24455            .build()
24456            .expect("client");
24457        let mut worker = Worker::new(client, "rust-workers")
24458            .worker_id("capacity-worker")
24459            .poll_timeout(Duration::from_millis(10))
24460            .retry_policy(WorkerRetryPolicy {
24461                max_retries: 0,
24462                initial_backoff: Duration::from_millis(1),
24463                max_backoff: Duration::from_millis(1),
24464            });
24465        worker.register_activity("capacity.activity", |_ctx, _input| async move {
24466            Ok(json!({"handled": true}))
24467        });
24468
24469        worker
24470            .run_until(tokio::time::sleep(Duration::from_millis(50)))
24471            .await
24472            .expect("capacity backpressure must not stop the worker");
24473
24474        assert!(
24475            server.request_count("/api/worker/activity-tasks/poll") >= 2,
24476            "the activity poller must continue after capacity backpressure"
24477        );
24478        assert_eq!(
24479            server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24480            1,
24481            "the worker must complete work returned after capacity recovers"
24482        );
24483    }
24484
24485    #[test]
24486    fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
24487        let capacity = Error::Http {
24488            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24489            body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
24490        };
24491        assert_eq!(
24492            worker_poll_capacity_retry_after(&capacity),
24493            Some(Duration::from_secs(3))
24494        );
24495
24496        let rejected_capacity = Error::Http {
24497            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24498            body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
24499        };
24500        assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
24501        assert!(!worker_operation_is_retryable(&rejected_capacity));
24502
24503        let ordinary_rate_limit = Error::Http {
24504            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24505            body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
24506                .to_string(),
24507        };
24508        assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
24509        assert!(worker_operation_is_retryable(&ordinary_rate_limit));
24510    }
24511
24512    #[tokio::test]
24513    async fn worker_bounds_transport_retries() {
24514        let server = MockWorkerServer::unavailable_polls();
24515        let client = Client::builder(server.base_url())
24516            .timeout(Duration::from_secs(2))
24517            .build()
24518            .expect("client");
24519        let mut worker = Worker::new(client, "rust-workers")
24520            .worker_id("bounded-retry-worker")
24521            .poll_timeout(Duration::from_millis(10))
24522            .retry_policy(WorkerRetryPolicy {
24523                max_retries: 2,
24524                initial_backoff: Duration::from_millis(1),
24525                max_backoff: Duration::from_millis(1),
24526            });
24527        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24528
24529        let error = worker.run().await.expect_err("retry bound must terminate");
24530        assert!(matches!(error, Error::Transport(_)));
24531        assert_eq!(
24532            server.request_count("/api/worker/workflow-tasks/poll"),
24533            3,
24534            "one initial request plus exactly two retries"
24535        );
24536    }
24537
24538    #[tokio::test]
24539    async fn worker_retry_policy_can_disable_poll_retries() {
24540        let server = MockWorkerServer::unavailable_polls();
24541        let client = Client::builder(server.base_url())
24542            .timeout(Duration::from_secs(2))
24543            .build()
24544            .expect("client");
24545        let mut worker = Worker::new(client, "rust-workers")
24546            .worker_id("no-retry-worker")
24547            .poll_timeout(Duration::from_millis(10))
24548            .retry_policy(WorkerRetryPolicy {
24549                max_retries: 0,
24550                initial_backoff: Duration::from_millis(1),
24551                max_backoff: Duration::from_millis(1),
24552            });
24553        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24554
24555        let error = worker
24556            .run_once()
24557            .await
24558            .expect_err("disabled retries must return the first transport failure");
24559        assert!(matches!(error, Error::Transport(_)));
24560        assert_eq!(
24561            server.request_count("/api/worker/workflow-tasks/poll"),
24562            1,
24563            "max_retries=0 must send only the initial request"
24564        );
24565    }
24566
24567    #[tokio::test]
24568    async fn worker_does_not_retry_authentication_failures() {
24569        let server = MockWorkerServer::unauthorized_polls();
24570        let client = Client::builder(server.base_url())
24571            .timeout(Duration::from_secs(2))
24572            .build()
24573            .expect("client");
24574        let mut worker = Worker::new(client, "rust-workers")
24575            .worker_id("unauthorized-worker")
24576            .poll_timeout(Duration::from_millis(10));
24577        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24578
24579        let error = worker
24580            .run()
24581            .await
24582            .expect_err("authentication must terminate");
24583        let Error::Http { status, body } = error else {
24584            panic!("expected stable HTTP authentication error");
24585        };
24586        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
24587        assert!(body.contains("authentication_failed"));
24588        assert_eq!(
24589            server.request_count("/api/worker/workflow-tasks/poll"),
24590            1,
24591            "authentication failures must not be retried"
24592        );
24593    }
24594
24595    #[derive(Clone, Debug)]
24596    struct CapturedRequest {
24597        headers: String,
24598        method: String,
24599        path: String,
24600        authorization: Option<String>,
24601        namespace: Option<String>,
24602        worker_protocol: Option<String>,
24603        control_protocol: Option<String>,
24604        body: String,
24605        received_at: Instant,
24606    }
24607
24608    struct MockWorkerServer {
24609        addr: SocketAddr,
24610        stop: Arc<AtomicBool>,
24611        requests: Arc<Mutex<Vec<CapturedRequest>>>,
24612        thread: Option<thread::JoinHandle<()>>,
24613    }
24614
24615    type RequestOverride = fn(&str, &str, usize) -> Option<(&'static str, String)>;
24616
24617    #[derive(Clone, Copy, Default)]
24618    struct MockWorkerBehavior {
24619        response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
24620        request_override: Option<RequestOverride>,
24621        storage_refusals: usize,
24622        storage_path: Option<&'static str>,
24623        storage_unavailable: bool,
24624        storage_mid_poll: bool,
24625        storage_activity: bool,
24626        storage_query: bool,
24627        storage_wrong_poll_id: bool,
24628        reject_query_protocol: bool,
24629        reject_query_completion: bool,
24630        waiting_query_worker: bool,
24631        decline_registration: bool,
24632        complete_named_signal: bool,
24633        poll_failures_per_path: usize,
24634        long_poll_capacity_responses_per_path: usize,
24635        heartbeat_failures: usize,
24636        heartbeat_failure_request: Option<usize>,
24637        delayed_heartbeat_request: Option<usize>,
24638        heartbeat_response_delay: Duration,
24639        concurrent_requests: bool,
24640        unauthorized_polls: bool,
24641        reject_registration: bool,
24642        reject_registration_protocol: bool,
24643        reject_deregistration: bool,
24644        reject_deregistration_protocol: bool,
24645        cancelled_activity: bool,
24646        draining_polls: bool,
24647        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
24648        workflow_completion_status: Option<&'static str>,
24649        workflow_completion_body: Option<&'static str>,
24650    }
24651
24652    impl MockWorkerServer {
24653        fn start() -> Self {
24654            Self::start_with_behavior(MockWorkerBehavior::default())
24655        }
24656
24657        fn reject_query_protocol() -> Self {
24658            Self::start_with_behavior(MockWorkerBehavior {
24659                reject_query_protocol: true,
24660                ..MockWorkerBehavior::default()
24661            })
24662        }
24663
24664        fn reject_query_completion() -> Self {
24665            Self::start_with_behavior(MockWorkerBehavior {
24666                reject_query_completion: true,
24667                ..MockWorkerBehavior::default()
24668            })
24669        }
24670
24671        fn waiting_query_worker() -> Self {
24672            Self::start_with_behavior(MockWorkerBehavior {
24673                waiting_query_worker: true,
24674                complete_named_signal: true,
24675                ..MockWorkerBehavior::default()
24676            })
24677        }
24678
24679        fn transient_worker_failures() -> Self {
24680            Self::start_with_behavior(MockWorkerBehavior {
24681                poll_failures_per_path: 1,
24682                heartbeat_failures: 1,
24683                ..MockWorkerBehavior::default()
24684            })
24685        }
24686
24687        fn consecutive_poll_failures(count: usize) -> Self {
24688            Self::start_with_behavior(MockWorkerBehavior {
24689                poll_failures_per_path: count,
24690                ..MockWorkerBehavior::default()
24691            })
24692        }
24693
24694        fn capacity_limited_activity_poll() -> Self {
24695            Self::start_with_behavior(MockWorkerBehavior {
24696                long_poll_capacity_responses_per_path: 1,
24697                ..MockWorkerBehavior::default()
24698            })
24699        }
24700
24701        fn delayed_heartbeat_worker() -> Self {
24702            Self::start_with_behavior(MockWorkerBehavior {
24703                waiting_query_worker: true,
24704                delayed_heartbeat_request: Some(2),
24705                heartbeat_response_delay: Duration::from_millis(1_500),
24706                concurrent_requests: true,
24707                cancelled_activity: true,
24708                ..MockWorkerBehavior::default()
24709            })
24710        }
24711
24712        fn heartbeat_retry_worker() -> Self {
24713            Self::start_with_behavior(MockWorkerBehavior {
24714                waiting_query_worker: true,
24715                heartbeat_failure_request: Some(2),
24716                concurrent_requests: true,
24717                ..MockWorkerBehavior::default()
24718            })
24719        }
24720
24721        fn unavailable_polls() -> Self {
24722            Self::start_with_behavior(MockWorkerBehavior {
24723                poll_failures_per_path: usize::MAX,
24724                ..MockWorkerBehavior::default()
24725            })
24726        }
24727
24728        fn unauthorized_polls() -> Self {
24729            Self::start_with_behavior(MockWorkerBehavior {
24730                unauthorized_polls: true,
24731                ..MockWorkerBehavior::default()
24732            })
24733        }
24734
24735        fn rejected_registration() -> Self {
24736            Self::start_with_behavior(MockWorkerBehavior {
24737                reject_registration: true,
24738                ..MockWorkerBehavior::default()
24739            })
24740        }
24741
24742        fn rejected_registration_protocol() -> Self {
24743            Self::start_with_behavior(MockWorkerBehavior {
24744                reject_registration_protocol: true,
24745                ..MockWorkerBehavior::default()
24746            })
24747        }
24748
24749        fn declined_registration() -> Self {
24750            Self::start_with_behavior(MockWorkerBehavior {
24751                decline_registration: true,
24752                ..MockWorkerBehavior::default()
24753            })
24754        }
24755
24756        fn rejected_deregistration() -> Self {
24757            Self::start_with_behavior(MockWorkerBehavior {
24758                reject_deregistration: true,
24759                ..MockWorkerBehavior::default()
24760            })
24761        }
24762
24763        fn rejected_deregistration_protocol() -> Self {
24764            Self::start_with_behavior(MockWorkerBehavior {
24765                reject_deregistration_protocol: true,
24766                ..MockWorkerBehavior::default()
24767            })
24768        }
24769
24770        fn unauthorized_polls_and_rejected_deregistration() -> Self {
24771            Self::start_with_behavior(MockWorkerBehavior {
24772                unauthorized_polls: true,
24773                reject_deregistration: true,
24774                ..MockWorkerBehavior::default()
24775            })
24776        }
24777
24778        fn cancelled_activity() -> Self {
24779            Self::start_with_behavior(MockWorkerBehavior {
24780                cancelled_activity: true,
24781                ..MockWorkerBehavior::default()
24782            })
24783        }
24784
24785        fn draining_polls() -> Self {
24786            Self::start_with_behavior(MockWorkerBehavior {
24787                draining_polls: true,
24788                ..MockWorkerBehavior::default()
24789            })
24790        }
24791
24792        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
24793            Self::start_with_behavior(MockWorkerBehavior {
24794                invalid_task_payload_codec: Some(codec),
24795                ..MockWorkerBehavior::default()
24796            })
24797        }
24798
24799        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
24800            Self::start_with_behavior(MockWorkerBehavior {
24801                workflow_completion_status: Some(status),
24802                workflow_completion_body: Some(body),
24803                ..MockWorkerBehavior::default()
24804            })
24805        }
24806
24807        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
24808            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
24809            listener
24810                .set_nonblocking(true)
24811                .expect("configure mock listener");
24812            let addr = listener.local_addr().expect("mock server address");
24813            let stop = Arc::new(AtomicBool::new(false));
24814            let server_stop = Arc::clone(&stop);
24815            let requests = Arc::new(Mutex::new(Vec::new()));
24816            let server_requests = Arc::clone(&requests);
24817            let thread = thread::spawn(move || {
24818                let mut request_threads = Vec::new();
24819                while !server_stop.load(Ordering::SeqCst) {
24820                    match listener.accept() {
24821                        Ok((mut stream, _)) => {
24822                            if behavior.concurrent_requests {
24823                                let requests = Arc::clone(&server_requests);
24824                                request_threads.push(thread::spawn(move || {
24825                                    handle_mock_worker_request(&mut stream, &requests, behavior)
24826                                }));
24827                            } else {
24828                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
24829                            }
24830                        }
24831                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
24832                            let mut index = 0;
24833                            while index < request_threads.len() {
24834                                if request_threads[index].is_finished() {
24835                                    request_threads
24836                                        .swap_remove(index)
24837                                        .join()
24838                                        .expect("join mock request");
24839                                } else {
24840                                    index += 1;
24841                                }
24842                            }
24843                            thread::sleep(Duration::from_millis(5));
24844                        }
24845                        Err(_) => break,
24846                    }
24847                }
24848                for request_thread in request_threads {
24849                    request_thread.join().expect("join mock request");
24850                }
24851            });
24852
24853            Self {
24854                addr,
24855                stop,
24856                requests,
24857                thread: Some(thread),
24858            }
24859        }
24860
24861        fn base_url(&self) -> String {
24862            format!("http://{}", self.addr)
24863        }
24864
24865        fn worker_protocol_for(&self, path: &str) -> Option<String> {
24866            self.requests
24867                .lock()
24868                .expect("captured requests")
24869                .iter()
24870                .find(|request| request.path == path)
24871                .and_then(|request| request.worker_protocol.clone())
24872        }
24873
24874        fn control_protocol_for(&self, path: &str) -> Option<String> {
24875            self.requests
24876                .lock()
24877                .expect("captured requests")
24878                .iter()
24879                .find(|request| request.path == path)
24880                .and_then(|request| request.control_protocol.clone())
24881        }
24882
24883        fn method_for(&self, path: &str) -> Option<String> {
24884            self.requests
24885                .lock()
24886                .expect("captured requests")
24887                .iter()
24888                .find(|request| request.path == path)
24889                .map(|request| request.method.clone())
24890        }
24891
24892        fn authorization_for(&self, path: &str) -> Option<String> {
24893            self.requests
24894                .lock()
24895                .expect("captured requests")
24896                .iter()
24897                .find(|request| request.path == path)
24898                .and_then(|request| request.authorization.clone())
24899        }
24900
24901        fn namespace_for(&self, path: &str) -> Option<String> {
24902            self.requests
24903                .lock()
24904                .expect("captured requests")
24905                .iter()
24906                .find(|request| request.path == path)
24907                .and_then(|request| request.namespace.clone())
24908        }
24909
24910        fn request_count(&self, path: &str) -> usize {
24911            self.requests
24912                .lock()
24913                .expect("captured requests")
24914                .iter()
24915                .filter(|request| request.path == path)
24916                .count()
24917        }
24918
24919        fn captured_paths(&self) -> Vec<String> {
24920            self.requests
24921                .lock()
24922                .expect("captured requests")
24923                .iter()
24924                .map(|request| request.path.clone())
24925                .collect()
24926        }
24927
24928        fn request_times(&self, path: &str) -> Vec<Instant> {
24929            self.requests
24930                .lock()
24931                .expect("captured requests")
24932                .iter()
24933                .filter(|request| request.path == path)
24934                .map(|request| request.received_at)
24935                .collect()
24936        }
24937
24938        fn request_body(&self, path: &str) -> Value {
24939            let requests = self.requests.lock().expect("captured requests");
24940            let body = &requests
24941                .iter()
24942                .find(|request| request.path == path)
24943                .unwrap_or_else(|| panic!("missing request for {path}"))
24944                .body;
24945            serde_json::from_str(body).unwrap_or_else(|error| {
24946                panic!("invalid JSON request body for {path}: {error}: {body:?}")
24947            })
24948        }
24949
24950        fn request_bodies(&self, path: &str) -> Vec<Value> {
24951            self.requests
24952                .lock()
24953                .expect("captured requests")
24954                .iter()
24955                .filter(|request| request.path == path)
24956                .map(|request| {
24957                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
24958                        panic!(
24959                            "invalid JSON request body for {path}: {error}: {:?}",
24960                            request.body
24961                        )
24962                    })
24963                })
24964                .collect()
24965        }
24966    }
24967
24968    impl Drop for MockWorkerServer {
24969        fn drop(&mut self) {
24970            self.stop.store(true, Ordering::SeqCst);
24971            let _ = TcpStream::connect(self.addr);
24972
24973            if let Some(thread) = self.thread.take() {
24974                thread.join().expect("join mock server");
24975            }
24976        }
24977    }
24978
24979    fn handle_mock_worker_request(
24980        stream: &mut TcpStream,
24981        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
24982        behavior: MockWorkerBehavior,
24983    ) {
24984        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
24985        let mut buffer = [0_u8; 8192];
24986        let mut request = Vec::new();
24987
24988        loop {
24989            match stream.read(&mut buffer) {
24990                Ok(0) => break,
24991                Ok(read) => {
24992                    request.extend_from_slice(&buffer[..read]);
24993                    if mock_request_is_complete(&request) {
24994                        break;
24995                    }
24996                }
24997                Err(error)
24998                    if matches!(
24999                        error.kind(),
25000                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
25001                    ) =>
25002                {
25003                    break;
25004                }
25005                Err(_) => return,
25006            }
25007        }
25008
25009        let request = String::from_utf8_lossy(&request);
25010        let body = request
25011            .split_once("\r\n\r\n")
25012            .map(|(_, body)| body)
25013            .unwrap_or_default();
25014        let path = request
25015            .lines()
25016            .next()
25017            .and_then(|line| line.split_whitespace().nth(1))
25018            .unwrap_or_default();
25019        let method = request
25020            .lines()
25021            .next()
25022            .and_then(|line| line.split_whitespace().next())
25023            .unwrap_or_default();
25024        let authorization = request.lines().find_map(|line| {
25025            let (name, value) = line.split_once(':')?;
25026            name.eq_ignore_ascii_case("Authorization")
25027                .then(|| value.trim().to_string())
25028        });
25029        let namespace = request.lines().find_map(|line| {
25030            let (name, value) = line.split_once(':')?;
25031            name.eq_ignore_ascii_case("X-Namespace")
25032                .then(|| value.trim().to_string())
25033        });
25034        let worker_protocol = request.lines().find_map(|line| {
25035            let (name, value) = line.split_once(':')?;
25036            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
25037                .then(|| value.trim().to_string())
25038        });
25039        let control_protocol = request.lines().find_map(|line| {
25040            let (name, value) = line.split_once(':')?;
25041            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
25042                .then(|| value.trim().to_string())
25043        });
25044        let request_number = {
25045            let mut requests = requests.lock().expect("captured requests");
25046            requests.push(CapturedRequest {
25047                headers: request
25048                    .split_once("\r\n\r\n")
25049                    .map_or("", |(headers, _)| headers)
25050                    .to_owned(),
25051                method: method.to_string(),
25052                path: path.to_string(),
25053                authorization,
25054                namespace,
25055                worker_protocol: worker_protocol.clone(),
25056                control_protocol,
25057                body: body.to_string(),
25058                received_at: Instant::now(),
25059            });
25060            requests
25061                .iter()
25062                .filter(|request| request.path == path)
25063                .count()
25064        };
25065
25066        if let Some(response) = behavior
25067            .request_override
25068            .and_then(|handler| handler(path, body, request_number))
25069        {
25070            write_mock_response(stream, response.0, &response.1);
25071            return;
25072        }
25073        if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
25074            write_mock_response(stream, response.0, &response.1);
25075            return;
25076        }
25077        if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
25078            return;
25079        }
25080        let pressure_path = behavior
25081            .storage_path
25082            .is_some_and(|part| path.contains(part));
25083        let prior_failures = if path.ends_with("/poll") {
25084            behavior.poll_failures_per_path
25085        } else {
25086            0
25087        };
25088        if pressure_path
25089            && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
25090        {
25091            let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
25092            let poll_id = path
25093                .ends_with("/poll")
25094                .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
25095            let mut refusal = storage_refusal(
25096                poll_id,
25097                behavior.storage_unavailable,
25098                behavior.storage_mid_poll,
25099            );
25100            if behavior.storage_wrong_poll_id {
25101                refusal["poll_request_id"] = json!("wrong-poll");
25102            }
25103            write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
25104            return;
25105        }
25106        if path.contains("/storage-task/")
25107            || path.contains("/storage-activity/")
25108            || path.contains("/storage-query/")
25109        {
25110            write_mock_response(stream, "200 OK", "{}");
25111            return;
25112        }
25113        if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
25114            write_mock_response(stream, "200 OK", &json!({"task":{
25115                "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
25116                "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
25117                "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25118                "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25119                "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
25120            }}).to_string());
25121            return;
25122        }
25123        if behavior.storage_activity
25124            && path == "/api/worker/activity-tasks/poll"
25125            && request_number == 1
25126        {
25127            write_mock_response(stream, "200 OK", &json!({"task":{
25128                "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
25129                "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25130                "attempt_number":7, "lease_owner":"storage-worker"
25131            }}).to_string());
25132            return;
25133        }
25134
25135        if path == "/api/worker/register" {
25136            if behavior.reject_registration_protocol {
25137                write_mock_response(
25138                    stream,
25139                    "400 Bad Request",
25140                    r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
25141                );
25142                return;
25143            }
25144            if behavior.reject_registration {
25145                write_mock_response(
25146                    stream,
25147                    "503 Service Unavailable",
25148                    r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
25149                );
25150                return;
25151            }
25152        }
25153
25154        if path.starts_with("/api/worker/registrations/") {
25155            if behavior.reject_deregistration_protocol {
25156                write_mock_response(
25157                    stream,
25158                    "400 Bad Request",
25159                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
25160                );
25161            } else if behavior.reject_deregistration {
25162                write_mock_response(
25163                    stream,
25164                    "403 Forbidden",
25165                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
25166                );
25167            } else {
25168                write_mock_response(
25169                    stream,
25170                    "200 OK",
25171                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
25172                );
25173            }
25174            return;
25175        }
25176
25177        let is_poll = matches!(
25178            path,
25179            "/api/worker/workflow-tasks/poll"
25180                | "/api/worker/activity-tasks/poll"
25181                | "/api/worker/query-tasks/poll"
25182        );
25183        if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
25184            write_mock_response(
25185                stream,
25186                "429 Too Many Requests",
25187                r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
25188            );
25189            return;
25190        }
25191        if is_poll && request_number <= behavior.poll_failures_per_path {
25192            return;
25193        }
25194        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
25195            return;
25196        }
25197        if path == "/api/worker/heartbeat"
25198            && behavior.heartbeat_failure_request == Some(request_number)
25199        {
25200            return;
25201        }
25202        if path == "/api/worker/heartbeat"
25203            && behavior.delayed_heartbeat_request == Some(request_number)
25204        {
25205            thread::sleep(behavior.heartbeat_response_delay);
25206        }
25207        if behavior.unauthorized_polls && is_poll {
25208            write_mock_response(
25209                stream,
25210                "401 Unauthorized",
25211                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
25212            );
25213            return;
25214        }
25215        if behavior.draining_polls && is_poll {
25216            write_mock_response(
25217                stream,
25218                "409 Conflict",
25219                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
25220            );
25221            return;
25222        }
25223
25224        if let Some(codec_case) = behavior.invalid_task_payload_codec {
25225            if is_poll && request_number == 1 {
25226                let mut task = match path {
25227                    "/api/worker/workflow-tasks/poll" => json!({
25228                        "task_id": "codec-workflow",
25229                        "workflow_type": "codec.workflow",
25230                        "payload_codec": DEFAULT_CODEC,
25231                        "workflow_task_attempt": 1,
25232                        "lease_owner": "codec-worker"
25233                    }),
25234                    "/api/worker/activity-tasks/poll" => json!({
25235                        "task_id": "codec-activity",
25236                        "activity_attempt_id": "codec-activity-attempt",
25237                        "activity_type": "codec.activity",
25238                        "payload_codec": DEFAULT_CODEC,
25239                        "attempt_number": 1,
25240                        "lease_owner": "codec-worker"
25241                    }),
25242                    "/api/worker/query-tasks/poll" => json!({
25243                        "query_task_id": "codec-query",
25244                        "query_task_attempt": 1,
25245                        "workflow_type": "codec.workflow",
25246                        "query_name": "known",
25247                        "payload_codec": DEFAULT_CODEC,
25248                        "lease_owner": "codec-worker"
25249                    }),
25250                    _ => unreachable!("is_poll limits task codec probe paths"),
25251                };
25252                codec_case.apply(&mut task);
25253                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
25254                return;
25255            }
25256
25257            if matches!(
25258                path,
25259                "/api/worker/workflow-tasks/codec-workflow/fail"
25260                    | "/api/worker/activity-tasks/codec-activity/fail"
25261                    | "/api/worker/query-tasks/codec-query/fail"
25262            ) {
25263                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
25264                return;
25265            }
25266        }
25267
25268        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
25269            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
25270            let body = format!(
25271                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
25272            );
25273            write_mock_response(stream, "400 Bad Request", &body);
25274            return;
25275        }
25276
25277        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
25278        {
25279            write_mock_response(
25280                stream,
25281                "409 Conflict",
25282                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
25283            );
25284            return;
25285        }
25286
25287        if behavior.workflow_completion_status.is_some()
25288            && path == "/api/worker/workflow-tasks/poll"
25289            && request_number == 1
25290        {
25291            write_mock_response(
25292                stream,
25293                "200 OK",
25294                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
25295            );
25296            return;
25297        }
25298
25299        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
25300            if let (Some(status), Some(body)) = (
25301                behavior.workflow_completion_status,
25302                behavior.workflow_completion_body,
25303            ) {
25304                write_mock_response(stream, status, body);
25305                return;
25306            }
25307        }
25308
25309        if behavior.waiting_query_worker {
25310            if behavior.complete_named_signal
25311                && path == "/api/worker/workflow-tasks/poll"
25312                && request_number == 1
25313            {
25314                let body = json!({
25315                    "task": {
25316                        "task_id": "snapshot-open",
25317                        "workflow_id": "snapshot-1",
25318                        "run_id": "snapshot-run-1",
25319                        "workflow_type": "snapshot",
25320                        "payload_codec": DEFAULT_CODEC,
25321                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25322                            .expect("Avro workflow arguments"),
25323                        "history_events": [],
25324                        "workflow_task_attempt": 1,
25325                        "lease_owner": "rust-snapshot-worker"
25326                    }
25327                })
25328                .to_string();
25329                write_mock_response(stream, "200 OK", &body);
25330                return;
25331            }
25332
25333            let signal_request = request_number - usize::from(behavior.complete_named_signal);
25334            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25335            if path == "/api/worker/workflow-tasks/poll"
25336                && signal_request >= 1
25337                && signal_request <= signal_request_limit
25338            {
25339                let finish = behavior.complete_named_signal && signal_request == 3;
25340                let amounts = if signal_request == 1 {
25341                    vec![3]
25342                } else {
25343                    vec![3, 5]
25344                };
25345                let task_id = if signal_request == 1 {
25346                    "snapshot-wait-3"
25347                } else if finish {
25348                    "snapshot-finish"
25349                } else {
25350                    "snapshot-wait-5"
25351                };
25352                let mut history_events = std::iter::once(json!({
25353                    "event_type": "SignalWaitOpened",
25354                    "payload": {"sequence": 1, "signal_name": "finish"}
25355                }))
25356                .chain(amounts.iter().enumerate().map(|(index, amount)| {
25357                    json!({
25358                        "event_type": "SignalReceived",
25359                        "payload": {
25360                            "signal_id": format!("increment-{amount}"),
25361                            "signal_name": "increment",
25362                            "workflow_sequence": index + 2,
25363                            "payload_codec": DEFAULT_CODEC,
25364                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25365                                .expect("Avro signal envelope")
25366                        }
25367                    })
25368                }))
25369                .collect::<Vec<_>>();
25370                let (resume_id, resume_name, resume_arguments) = if finish {
25371                    history_events.push(json!({
25372                        "event_type": "SignalReceived",
25373                        "payload": {
25374                            "signal_id": "finish",
25375                            "signal_name": "finish",
25376                            "workflow_sequence": 4,
25377                            "payload_codec": DEFAULT_CODEC,
25378                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25379                                .expect("Avro finish signal envelope")
25380                        }
25381                    }));
25382                    (
25383                        "finish".to_string(),
25384                        "finish".to_string(),
25385                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
25386                            .expect("Avro finish resume signal"),
25387                    )
25388                } else {
25389                    let amount = amounts.last().expect("amount");
25390                    (
25391                        format!("increment-{amount}"),
25392                        "increment".to_string(),
25393                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25394                            .expect("Avro increment resume signal"),
25395                    )
25396                };
25397                let body = json!({
25398                    "task": {
25399                        "task_id": task_id,
25400                        "workflow_id": "snapshot-1",
25401                        "run_id": "snapshot-run-1",
25402                        "workflow_type": "snapshot",
25403                        "payload_codec": DEFAULT_CODEC,
25404                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25405                            .expect("Avro workflow arguments"),
25406                        "history_events": history_events,
25407                        "workflow_task_attempt": 1,
25408                        "workflow_signal_id": resume_id,
25409                        "signal_name": resume_name,
25410                        "signal_arguments": resume_arguments,
25411                        "lease_owner": "rust-snapshot-worker"
25412                    }
25413                })
25414                .to_string();
25415                write_mock_response(stream, "200 OK", &body);
25416                return;
25417            }
25418
25419            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25420                let history_events = [3, 5]
25421                    .into_iter()
25422                    .enumerate()
25423                    .map(|(index, amount)| {
25424                        json!({
25425                            "event_type": "SignalReceived",
25426                            "payload": {
25427                                "signal_id": format!("increment-{amount}"),
25428                                "signal_name": "increment",
25429                                "workflow_sequence": index + 2,
25430                                "payload_codec": DEFAULT_CODEC,
25431                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25432                                    .expect("Avro query signal envelope")
25433                            }
25434                        })
25435                    })
25436                    .collect::<Vec<_>>();
25437                let body = json!({
25438                    "task": {
25439                        "query_task_id": "snapshot-current",
25440                        "query_task_attempt": 1,
25441                        "lease_owner": "rust-snapshot-worker",
25442                        "workflow_id": "snapshot-1",
25443                        "run_id": "snapshot-run-1",
25444                        "workflow_type": "snapshot",
25445                        "query_name": "current",
25446                        "payload_codec": DEFAULT_CODEC,
25447                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25448                            .expect("Avro workflow arguments"),
25449                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25450                            .expect("Avro query arguments"),
25451                        "history_events": history_events,
25452                        "run_status": "waiting"
25453                    }
25454                })
25455                .to_string();
25456                write_mock_response(stream, "200 OK", &body);
25457                return;
25458            }
25459
25460            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25461                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25462            {
25463                write_mock_response(
25464                    stream,
25465                    "200 OK",
25466                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
25467                );
25468                return;
25469            }
25470
25471            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25472                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25473                return;
25474            }
25475
25476            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25477                write_mock_response(
25478                    stream,
25479                    "200 OK",
25480                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25481                );
25482                return;
25483            }
25484
25485            if path == "/api/worker/query-tasks/snapshot-current/complete" {
25486                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
25487                return;
25488            }
25489        }
25490
25491        if matches!(
25492            path,
25493            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
25494        ) {
25495            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25496                .expect("typed mock result");
25497            let body = json!({
25498                "result": typed_fidelity_probe().into_json().expect("result projection"),
25499                "result_envelope": result,
25500            })
25501            .to_string();
25502            write_mock_response(stream, "200 OK", &body);
25503            return;
25504        }
25505
25506        if path == "/api/workflows/typed-1" {
25507            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25508                .expect("typed mock result");
25509            let body = json!({
25510                "workflow_id": "typed-1",
25511                "run_id": "run-typed-1",
25512                "workflow_type": "typed.echo",
25513                "status": "completed",
25514                "output": typed_fidelity_probe().into_json().expect("output projection"),
25515                "output_envelope": result,
25516            })
25517            .to_string();
25518            write_mock_response(stream, "200 OK", &body);
25519            return;
25520        }
25521
25522        let (status, body) = match path {
25523            "/api/cluster/info" => ("200 OK", r#"{"limits":{"max_payload_bytes":2097152}}"#),
25524            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
25525            "/api/workflows" => (
25526                "201 Created",
25527                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
25528            ),
25529            "/api/worker/register" if behavior.decline_registration => (
25530                "200 OK",
25531                r#"{"worker_id":"declined-worker","registered":false}"#,
25532            ),
25533            "/api/worker/register" if behavior.waiting_query_worker => (
25534                "200 OK",
25535                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
25536            ),
25537            "/api/worker/register" => (
25538                "200 OK",
25539                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
25540            ),
25541            "/api/worker/heartbeat" => ("200 OK", "{}"),
25542            "/api/worker/activity-tasks/poll"
25543                if behavior.cancelled_activity && request_number == 1 =>
25544            {
25545                (
25546                    "200 OK",
25547                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
25548                )
25549            }
25550            "/api/worker/activity-tasks/poll"
25551                if behavior.long_poll_capacity_responses_per_path > 0
25552                    && request_number
25553                        == behavior
25554                            .long_poll_capacity_responses_per_path
25555                            .saturating_add(1) =>
25556            {
25557                (
25558                    "200 OK",
25559                    r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
25560                )
25561            }
25562            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
25563                ("200 OK", r#"{"task":null}"#)
25564            }
25565            "/api/worker/query-tasks/poll"
25566                if behavior.reject_query_completion && request_number == 1 =>
25567            {
25568                (
25569                    "200 OK",
25570                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
25571                )
25572            }
25573            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
25574            "/api/worker/query-tasks/query-capture/complete"
25575            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
25576            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
25577                "200 OK",
25578                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
25579            ),
25580            "/api/worker/activity-tasks/activity-cancel/complete" => (
25581                "409 Conflict",
25582                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
25583            ),
25584            "/api/worker/activity-tasks/activity-typed/complete"
25585            | "/api/worker/activity-tasks/activity-typed/fail"
25586            | "/api/worker/activity-tasks/capacity-activity/complete"
25587            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
25588            "/api/workflows/counter-1/query/current" => (
25589                "200 OK",
25590                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
25591            ),
25592            "/api/workflows/counter-1/query/missing" => (
25593                "404 Not Found",
25594                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
25595            ),
25596            "/api/workflows/wf-lifecycle/cancel" => (
25597                "200 OK",
25598                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
25599            ),
25600            "/api/workflows/wf-lifecycle/terminate" => (
25601                "200 OK",
25602                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
25603            ),
25604            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
25605                "200 OK",
25606                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
25607            ),
25608            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
25609                "200 OK",
25610                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
25611            ),
25612            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
25613            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
25614                "409 Conflict",
25615                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
25616            ),
25617            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
25618                "200 OK",
25619                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
25620            ),
25621            "/api/workflows/wf-cancelled" => (
25622                "200 OK",
25623                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
25624            ),
25625            "/api/workflows/wf-terminated" => (
25626                "200 OK",
25627                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
25628            ),
25629            "/api/workflows/wf-timed-out" => (
25630                "200 OK",
25631                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
25632            ),
25633            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
25634                "200 OK",
25635                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
25636            ),
25637            "/api/workflows/wf-selected" => (
25638                "200 OK",
25639                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
25640            ),
25641            "/api/workflows/wf-selected/runs/run-selected" => (
25642                "200 OK",
25643                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
25644            ),
25645            _ => ("404 Not Found", r#"{"message":"not found"}"#),
25646        };
25647        write_mock_response(stream, status, body);
25648    }
25649
25650    fn mock_request_is_complete(request: &[u8]) -> bool {
25651        let Some(header_end) = request
25652            .windows(4)
25653            .position(|window| window == b"\r\n\r\n")
25654            .map(|position| position + 4)
25655        else {
25656            return false;
25657        };
25658        let headers = String::from_utf8_lossy(&request[..header_end]);
25659        let content_length = headers.lines().find_map(|line| {
25660            let (name, value) = line.split_once(':')?;
25661            name.eq_ignore_ascii_case("content-length")
25662                .then(|| value.trim().parse::<usize>().ok())
25663                .flatten()
25664        });
25665
25666        request.len() >= header_end + content_length.unwrap_or(0)
25667    }
25668
25669    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
25670        let response = format!(
25671            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
25672            body.len()
25673        );
25674
25675        let _ = stream.write_all(response.as_bytes());
25676        let _ = stream.flush();
25677    }
25678}