Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod runtime_payloads;
4
5use std::{
6    any::{type_name, Any, TypeId},
7    collections::{BTreeMap, HashMap},
8    future::Future,
9    io::{self, Read},
10    pin::Pin,
11    sync::{
12        atomic::{AtomicBool, Ordering},
13        Arc, Mutex, OnceLock,
14    },
15    task::{Context as TaskContext, Poll},
16    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
17};
18
19use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
20use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
21use chrono::DateTime;
22use futures_util::{future::OptionFuture, task::noop_waker_ref};
23use serde::{
24    de::DeserializeOwned,
25    ser::{SerializeMap, SerializeSeq},
26    Deserialize, Deserializer, Serialize, Serializer,
27};
28pub use serde_json::{json, Value};
29use sha2::{Digest, Sha256};
30use thiserror::Error;
31pub use uuid::Uuid;
32
33pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
34/// First additive worker protocol that defines portable worker-affinity features.
35pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
36pub const CONTROL_PLANE_VERSION: &str = "2";
37pub const DEFAULT_CODEC: &str = "avro";
38pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
39/// Worker-registration capability for authored condition-wait occurrence identity.
40pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
41    "condition_wait_occurrence_identity";
42/// Worker-registration capability for portable memo upserts.
43pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
44/// Worker-registration capability for server-routed read-only queries.
45pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
46/// Worker-registration capability for canonical typed search attributes.
47pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
48/// Worker-registration capability for synchronous workflow updates.
49pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
50/// Worker-registration capability for durable named input streams.
51pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
52/// Worker-registration capability for persisted first-completion selection.
53pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
54pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
55pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
56pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
57pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
58pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
59/// First additive worker protocol that defines query-task transport.
60pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
61/// First additive worker protocol that defines typed search-attribute upserts.
62pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
63/// First additive worker protocol that defines portable memo upserts.
64pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
65/// First additive worker protocol that preserves declared search-attribute types.
66pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
67/// First additive worker protocol that defines external durable condition waits.
68pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
69/// First additive worker protocol that preserves authored condition-wait occurrences.
70pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
71/// First additive worker protocol that defines durable selection groups.
72pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
73
74pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
75    let Some((major, minor)) = version.split_once('.') else {
76        return false;
77    };
78    major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
79}
80
81fn validate_user_signal_name(signal_name: &str) -> Result<()> {
82    if signal_name == MESSAGE_STREAM_SIGNAL {
83        return Err(Error::Codec(format!(
84            "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
85        )));
86    }
87    Ok(())
88}
89
90const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
91const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
92    "Workflow task waiting for scheduled history.";
93const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
94const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
95const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
96const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
97const MAX_MEMO_ENTRIES: usize = 100;
98const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
99const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
100
101const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
102    "lease_expired",
103    "query_task_not_found",
104    "query_task_not_leased",
105    "query_task_timed_out",
106];
107
108/// Truthful service-worker manifest for features this SDK currently refuses.
109pub fn portable_worker_affinity_capability_manifest() -> Value {
110    json!({
111        "local_activities": {
112            "supported": false,
113            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
114            "reason": "rust_worker_does_not_execute_record_local_activity",
115        },
116        "worker_sessions": {
117            "supported": false,
118            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
119            "reason": "rust_worker_has_no_typed_session_lifecycle",
120        },
121        "sticky_execution": {
122            "supported": false,
123            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
124            "reason": "rust_worker_uses_complete_durable_history_replay",
125        },
126    })
127}
128
129/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
130pub const AVRO_VALUE_SCHEMA_JSON: &str =
131    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
132pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
133pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
134const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
135
136static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
137static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
138    OnceLock::new();
139
140#[derive(Clone, Copy)]
141enum RequestProtocol {
142    ControlPlane,
143    Worker(&'static str),
144}
145
146pub type Result<T> = std::result::Result<T, Error>;
147
148#[derive(Debug, Error)]
149pub enum Error {
150    #[error("transport error: {0}")]
151    Transport(#[from] reqwest::Error),
152    #[error(
153        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
154    )]
155    InvalidBaseUrl,
156    #[error("json error: {0}")]
157    Json(#[from] serde_json::Error),
158    #[error("http {status}: {body}")]
159    Http {
160        status: reqwest::StatusCode,
161        body: String,
162    },
163    #[error("codec error: {0}")]
164    Codec(String),
165    #[error(transparent)]
166    QueryFailed(QueryFailure),
167    #[error(transparent)]
168    Protocol(ProtocolFailure),
169    #[error(transparent)]
170    NonDeterministicReplay(ReplayFailure),
171    #[error(transparent)]
172    ChildWorkflowFailed(ChildWorkflowFailure),
173    #[error(transparent)]
174    ActivityFailed(ActivityFailure),
175    #[error(transparent)]
176    ParallelFailed(ParallelFailure),
177    #[error(transparent)]
178    SagaCompensationFailed(SagaCompensationFailure),
179    #[error(transparent)]
180    InvalidParallelGroup(ParallelGroupError),
181    #[error(transparent)]
182    DurableOperationCancelled(DurableOperationCancelled),
183    #[error(transparent)]
184    WorkflowCancellationRequested(WorkflowCancellationRequested),
185    #[error(transparent)]
186    WorkflowCommandRejected(WorkflowCommandRejection),
187    #[error(transparent)]
188    WorkflowFailed(WorkflowTerminalOutcome),
189    #[error(transparent)]
190    WorkflowCancelled(WorkflowTerminalOutcome),
191    #[error(transparent)]
192    WorkflowTerminated(WorkflowTerminalOutcome),
193    #[error(transparent)]
194    WorkflowTimedOut(WorkflowTerminalOutcome),
195    #[error(transparent)]
196    ActivityTaskRejected(ActivityTaskRejection),
197    #[error("workflow handler {0:?} is not registered")]
198    WorkflowNotRegistered(String),
199    #[error("activity handler {0:?} is not registered")]
200    ActivityNotRegistered(String),
201    #[error(
202        "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
203    )]
204    HandlerType {
205        handler_kind: HandlerKind,
206        handler_name: String,
207        value_kind: HandlerValueKind,
208        rust_type: &'static str,
209        message: String,
210    },
211    #[error("workflow future yielded without emitting a durable command")]
212    WorkflowYieldedWithoutCommand,
213    #[error(
214        "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
215    )]
216    MissingWorkflowCommandIdentity,
217    #[error("workflow state lock is poisoned")]
218    WorkflowStatePoisoned,
219    #[error("timer duration is too large for the worker protocol")]
220    TimerDurationOverflow,
221    #[error(transparent)]
222    InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
223    #[error(transparent)]
224    InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
225    #[error("operation timed out")]
226    Timeout,
227    #[error(
228        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
229    )]
230    MissingRoleCredentials {
231        role: &'static str,
232        opposite_role: &'static str,
233    },
234    #[error("worker loop error: {0}")]
235    WorkerLoop(String),
236    #[error(
237        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
238    )]
239    UnsupportedUpdateValidators { workflow_type: String },
240    #[error("{primary}; worker deregistration also failed: {deregistration}")]
241    WorkerShutdown {
242        primary: Box<Error>,
243        deregistration: Box<Error>,
244    },
245    #[error("invalid child workflow options: {0}")]
246    InvalidChildWorkflowOptions(String),
247    #[error("invalid workflow memo update: {0}")]
248    InvalidMemoUpdate(String),
249    #[error(
250        "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
251    )]
252    WorkflowMemoUpdatesUnavailable,
253    #[error(transparent)]
254    InvalidActivityOptions(ActivityOptionsError),
255    #[error(transparent)]
256    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
257    #[doc(hidden)]
258    #[error("workflow requested continue as new")]
259    ContinueAsNew(ContinueAsNewRequest),
260}
261
262/// Validation failure for a durable condition-wait definition.
263#[derive(Clone, Debug, Error, PartialEq, Eq)]
264pub enum ConditionWaitOptionsError {
265    #[error("condition_key must be non-empty")]
266    EmptyKey,
267    #[error("condition_definition_fingerprint must be non-empty")]
268    EmptyPredicateIdentity,
269    #[error("condition timeout is too large for the worker protocol")]
270    TimeoutOverflow,
271}
272
273/// Stable identity and optional durable timeout for a condition wait.
274///
275/// `predicate_identity` is recorded as the worker protocol's
276/// `condition_definition_fingerprint` and must change whenever predicate
277/// behavior changes. Prefer the [`wait_condition!`] macro when the predicate
278/// is written inline; it derives this identity from the predicate tokens.
279#[derive(Clone, Debug, PartialEq, Eq)]
280pub struct ConditionWaitOptions {
281    condition_key: String,
282    predicate_identity: String,
283    timeout: Option<Duration>,
284}
285
286impl ConditionWaitOptions {
287    pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
288        Self {
289            condition_key: condition_key.into(),
290            predicate_identity: predicate_identity.into(),
291            timeout: None,
292        }
293    }
294
295    pub fn timeout(mut self, timeout: Duration) -> Self {
296        self.timeout = Some(timeout);
297        self
298    }
299
300    fn validate(
301        &self,
302    ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
303        let condition_key = self.condition_key.trim();
304        if condition_key.is_empty() {
305            return Err(ConditionWaitOptionsError::EmptyKey);
306        }
307        let predicate_identity = self.predicate_identity.trim();
308        if predicate_identity.is_empty() {
309            return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
310        }
311        let timeout_seconds = self
312            .timeout
313            .map(|timeout| {
314                timeout
315                    .as_secs()
316                    .checked_add(u64::from(timeout.subsec_nanos() > 0))
317                    .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
318            })
319            .transpose()?;
320
321        Ok(ValidatedConditionWaitOptions {
322            condition_key: condition_key.to_string(),
323            predicate_identity: predicate_identity.to_string(),
324            timeout_seconds,
325        })
326    }
327}
328
329#[derive(Clone, Debug, PartialEq, Eq)]
330struct ValidatedConditionWaitOptions {
331    condition_key: String,
332    predicate_identity: String,
333    timeout_seconds: Option<u64>,
334}
335
336const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
337
338/// Unambiguous terminal result of a durable condition wait.
339#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
340#[serde(rename_all = "snake_case")]
341pub enum ConditionWaitResult {
342    Satisfied,
343    TimedOut,
344}
345
346impl ConditionWaitResult {
347    pub fn is_satisfied(self) -> bool {
348        self == Self::Satisfied
349    }
350
351    pub fn is_timed_out(self) -> bool {
352        self == Self::TimedOut
353    }
354}
355
356/// Build the stable condition definition identity used by [`wait_condition!`].
357#[doc(hidden)]
358pub fn __condition_definition_fingerprint(source: &str) -> String {
359    let mut digest = Sha256::new();
360    digest.update(b"durable-workflow-rust.wait-condition.v1\0");
361    digest.update(source.as_bytes());
362    format!("sha256:{:x}", digest.finalize())
363}
364
365/// Create a durable condition wait whose predicate definition is fingerprinted
366/// from its inline Rust tokens.
367///
368/// The returned [`ConditionWaitCall`] must be awaited. The timeout form is
369/// `wait_condition!(ctx, "approval", timeout: duration, || predicate)`.
370#[macro_export]
371macro_rules! wait_condition {
372    ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
373        $ctx.wait_condition(
374            $crate::ConditionWaitOptions::new(
375                $key,
376                $crate::__condition_definition_fingerprint(concat!(
377                    module_path!(),
378                    "\0",
379                    stringify!($predicate)
380                )),
381            )
382            .timeout($timeout),
383            $predicate,
384        )
385    }};
386    ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
387        $ctx.wait_condition(
388            $crate::ConditionWaitOptions::new(
389                $key,
390                $crate::__condition_definition_fingerprint(concat!(
391                    module_path!(),
392                    "\0",
393                    stringify!($predicate)
394                )),
395            ),
396            $predicate,
397        )
398    }};
399}
400
401const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
402const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
403const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
404const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
405const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
406
407/// Validation failure for a typed workflow search-attribute update.
408#[derive(Clone, Debug, Error, PartialEq, Eq)]
409pub enum SearchAttributeUpdateError {
410    #[error("search-attribute update requires at least one attribute")]
411    Empty,
412    #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
413    InvalidKey(String),
414    #[error("search-attribute update exceeds the limit of 100 attributes")]
415    TooManyAttributes,
416    #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
417    ValueTooLong {
418        key: String,
419        kind: &'static str,
420        limit: usize,
421    },
422    #[error(
423        "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
424    )]
425    EmptyString(String),
426    #[error("search attribute {0:?} has a non-finite float value")]
427    NonFiniteFloat(String),
428    #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
429    InvalidDateTime(String),
430    #[error("search-attribute update exceeds the 65536-byte protocol limit")]
431    PayloadTooLarge,
432}
433
434/// One public typed search-attribute value.
435#[derive(Clone, Debug, PartialEq)]
436pub enum SearchAttributeValue {
437    String(String),
438    Keyword(String),
439    KeywordList(Vec<String>),
440    Int(i64),
441    Float(f64),
442    Bool(bool),
443    DateTime(String),
444    Delete,
445}
446
447impl SearchAttributeValue {
448    fn type_name(&self) -> Option<&'static str> {
449        match self {
450            Self::String(_) => Some("string"),
451            Self::Keyword(_) => Some("keyword"),
452            Self::KeywordList(_) => Some("keyword_list"),
453            Self::Int(_) => Some("int"),
454            Self::Float(_) => Some("float"),
455            Self::Bool(_) => Some("bool"),
456            Self::DateTime(_) => Some("datetime"),
457            Self::Delete => None,
458        }
459    }
460
461    fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
462        let normalize_string = |value: String, kind: &'static str, limit: usize| {
463            let value = value.trim().to_string();
464            if value.is_empty() {
465                return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
466            }
467            if value.len() > limit {
468                return Err(SearchAttributeUpdateError::ValueTooLong {
469                    key: key.to_string(),
470                    kind,
471                    limit,
472                });
473            }
474            Ok(value)
475        };
476
477        match self {
478            Self::String(value) => Ok(Self::String(normalize_string(
479                value,
480                "string",
481                MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
482            )?)),
483            Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
484                value,
485                "keyword",
486                MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
487            )?)),
488            Self::KeywordList(values) => {
489                let values = values
490                    .into_iter()
491                    .map(|value| {
492                        let value = value.trim().to_string();
493                        if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
494                            return Err(SearchAttributeUpdateError::ValueTooLong {
495                                key: key.to_string(),
496                                kind: "keyword-list entry",
497                                limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
498                            });
499                        }
500                        Ok(value)
501                    })
502                    .collect::<std::result::Result<Vec<_>, _>>()?;
503                Ok(Self::KeywordList(values))
504            }
505            Self::Float(value) if !value.is_finite() => {
506                Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
507            }
508            Self::DateTime(value) => {
509                let value =
510                    normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
511                if DateTime::parse_from_rfc3339(&value).is_err() {
512                    return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
513                }
514                Ok(Self::DateTime(value))
515            }
516            value => Ok(value),
517        }
518    }
519
520    fn into_json(self) -> Value {
521        match self {
522            Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
523                Value::String(value)
524            }
525            Self::KeywordList(values) => {
526                Value::Array(values.into_iter().map(Value::String).collect())
527            }
528            Self::Int(value) => json!(value),
529            Self::Float(value) => json!(value),
530            Self::Bool(value) => json!(value),
531            Self::Delete => Value::Null,
532        }
533    }
534}
535
536/// Validated typed workflow-side search-attribute mutation.
537#[derive(Clone, Debug, Default, PartialEq)]
538pub struct SearchAttributeUpdate {
539    attributes: BTreeMap<String, SearchAttributeValue>,
540}
541
542impl SearchAttributeUpdate {
543    pub fn new() -> Self {
544        Self::default()
545    }
546
547    pub fn set(
548        mut self,
549        key: impl Into<String>,
550        value: SearchAttributeValue,
551    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
552        let key = key.into();
553        validate_search_attribute_key(&key)?;
554        if !self.attributes.contains_key(&key)
555            && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
556        {
557            return Err(SearchAttributeUpdateError::TooManyAttributes);
558        }
559        self.attributes.insert(key.clone(), value.normalized(&key)?);
560        self.validate_size()?;
561        Ok(self)
562    }
563
564    pub fn string(
565        self,
566        key: impl Into<String>,
567        value: impl Into<String>,
568    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
569        self.set(key, SearchAttributeValue::String(value.into()))
570    }
571
572    pub fn keyword(
573        self,
574        key: impl Into<String>,
575        value: impl Into<String>,
576    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
577        self.set(key, SearchAttributeValue::Keyword(value.into()))
578    }
579
580    pub fn keyword_list<I, V>(
581        self,
582        key: impl Into<String>,
583        values: I,
584    ) -> std::result::Result<Self, SearchAttributeUpdateError>
585    where
586        I: IntoIterator<Item = V>,
587        V: Into<String>,
588    {
589        self.set(
590            key,
591            SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
592        )
593    }
594
595    pub fn int(
596        self,
597        key: impl Into<String>,
598        value: i64,
599    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
600        self.set(key, SearchAttributeValue::Int(value))
601    }
602
603    pub fn float(
604        self,
605        key: impl Into<String>,
606        value: f64,
607    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
608        self.set(key, SearchAttributeValue::Float(value))
609    }
610
611    pub fn bool(
612        self,
613        key: impl Into<String>,
614        value: bool,
615    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
616        self.set(key, SearchAttributeValue::Bool(value))
617    }
618
619    pub fn datetime(
620        self,
621        key: impl Into<String>,
622        value: impl Into<String>,
623    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
624        self.set(key, SearchAttributeValue::DateTime(value.into()))
625    }
626
627    pub fn delete(
628        self,
629        key: impl Into<String>,
630    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
631        self.set(key, SearchAttributeValue::Delete)
632    }
633
634    fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
635        let (attributes, _) = self.clone().into_wire_parts();
636        if serde_json::to_vec(&attributes)
637            .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
638            .unwrap_or(true)
639        {
640            return Err(SearchAttributeUpdateError::PayloadTooLarge);
641        }
642        Ok(())
643    }
644
645    fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
646        let mut attributes = serde_json::Map::new();
647        let mut attribute_types = BTreeMap::new();
648        for (key, value) in self.attributes {
649            if let Some(type_name) = value.type_name() {
650                attribute_types.insert(key.clone(), type_name.to_string());
651            }
652            attributes.insert(key, value.into_json());
653        }
654        (Value::Object(attributes), attribute_types)
655    }
656
657    fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
658        if self.attributes.is_empty() {
659            return Err(SearchAttributeUpdateError::Empty);
660        }
661        self.validate_size()
662    }
663}
664
665fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
666    let valid = !key.is_empty()
667        && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
668        && key
669            .bytes()
670            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
671    if valid {
672        Ok(())
673    } else {
674        Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
675    }
676}
677
678/// The registered handler family reported by [`Error::HandlerType`].
679#[derive(Clone, Copy, Debug, PartialEq, Eq)]
680pub enum HandlerKind {
681    Workflow,
682    Activity,
683}
684
685impl std::fmt::Display for HandlerKind {
686    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
687        formatter.write_str(match self {
688            Self::Workflow => "workflow",
689            Self::Activity => "activity",
690        })
691    }
692}
693
694/// Whether a typed handler failed to adapt its input or result.
695#[derive(Clone, Copy, Debug, PartialEq, Eq)]
696pub enum HandlerValueKind {
697    Input,
698    Result,
699}
700
701impl std::fmt::Display for HandlerValueKind {
702    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
703        formatter.write_str(match self {
704            Self::Input => "input",
705            Self::Result => "result",
706        })
707    }
708}
709
710/// The lifecycle command sent to a workflow execution.
711#[derive(Clone, Copy, Debug, PartialEq, Eq)]
712pub enum WorkflowCommandKind {
713    Cancel,
714    Terminate,
715}
716
717impl WorkflowCommandKind {
718    fn as_str(self) -> &'static str {
719        match self {
720            Self::Cancel => "cancel",
721            Self::Terminate => "terminate",
722        }
723    }
724}
725
726/// Optional structured fields for a cancellation or termination request.
727#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
728pub struct WorkflowCommandOptions {
729    #[serde(skip_serializing_if = "Option::is_none")]
730    pub reason: Option<String>,
731    #[serde(skip_serializing_if = "Option::is_none")]
732    pub request_id: Option<String>,
733}
734
735/// Server-enforced timeout policy for a workflow start.
736///
737/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
738/// only bounds how long the caller waits. A server deadline produces a terminal
739/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
740/// `run_timeout`.
741#[derive(Clone, Debug, PartialEq, Eq)]
742pub struct WorkflowStartOptions {
743    pub execution_timeout_seconds: u64,
744    pub run_timeout_seconds: u64,
745}
746
747impl Default for WorkflowStartOptions {
748    fn default() -> Self {
749        Self {
750            execution_timeout_seconds: 3600,
751            run_timeout_seconds: 600,
752        }
753    }
754}
755
756impl WorkflowStartOptions {
757    pub fn new() -> Self {
758        Self::default()
759    }
760
761    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
762        self.execution_timeout_seconds = seconds;
763        self
764    }
765
766    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
767        self.run_timeout_seconds = seconds;
768        self
769    }
770
771    fn validate(&self) -> Result<()> {
772        if self.execution_timeout_seconds == 0 {
773            return Err(Error::Codec(
774                "execution_timeout_seconds must be at least 1".to_string(),
775            ));
776        }
777        if self.run_timeout_seconds == 0 {
778            return Err(Error::Codec(
779                "run_timeout_seconds must be at least 1".to_string(),
780            ));
781        }
782        if self.run_timeout_seconds > self.execution_timeout_seconds {
783            return Err(Error::Codec(
784                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
785            ));
786        }
787
788        Ok(())
789    }
790}
791
792/// Optional routing overrides for a continue-as-new transition.
793///
794/// Omitted values retain the current workflow type and task queue. Server-owned
795/// instance metadata is not accepted here and is carried by the server.
796#[derive(Clone, Debug, Default, PartialEq, Eq)]
797pub struct ContinueAsNewOptions {
798    pub workflow_type: Option<String>,
799    pub task_queue: Option<String>,
800}
801
802impl ContinueAsNewOptions {
803    pub fn new() -> Self {
804        Self::default()
805    }
806
807    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
808        self.workflow_type = Some(workflow_type.into());
809        self
810    }
811
812    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
813        self.task_queue = Some(task_queue.into());
814        self
815    }
816
817    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
818        for (field, value) in [
819            ("workflow_type", self.workflow_type.as_deref()),
820            ("task_queue", self.task_queue.as_deref()),
821        ] {
822            if value.is_some_and(|value| value.trim().is_empty()) {
823                return Err(ContinueAsNewOptionsError {
824                    field,
825                    message: format!("{field} must not be empty"),
826                });
827            }
828        }
829        Ok(())
830    }
831}
832
833/// A stable validation error raised before a continue-as-new command is emitted.
834#[derive(Clone, Debug, Error, PartialEq, Eq)]
835#[error("invalid continue-as-new option {field}: {message}")]
836pub struct ContinueAsNewOptionsError {
837    pub field: &'static str,
838    pub message: String,
839}
840
841/// Public history-budget information attached to the current workflow task.
842#[derive(Clone, Debug, Default, PartialEq, Eq)]
843pub struct WorkflowHistoryBudget {
844    pub event_count: u64,
845    pub size_bytes: Option<u64>,
846    pub continue_as_new_recommended: bool,
847    pub pressure: Option<String>,
848}
849
850#[doc(hidden)]
851#[derive(Clone, Debug)]
852pub struct ContinueAsNewRequest {
853    arguments: AvroValue,
854    options: ContinueAsNewOptions,
855}
856
857impl WorkflowCommandOptions {
858    pub fn new() -> Self {
859        Self::default()
860    }
861
862    pub fn reason(mut self, reason: impl Into<String>) -> Self {
863        self.reason = Some(reason.into());
864        self
865    }
866
867    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
868        self.request_id = Some(request_id.into());
869        self
870    }
871}
872
873/// The accepted, machine-readable result of a lifecycle command.
874#[derive(Clone, Debug, PartialEq)]
875pub struct WorkflowCommandResult {
876    pub command: WorkflowCommandKind,
877    pub workflow_id: String,
878    pub run_id: Option<String>,
879    pub outcome: Option<String>,
880    pub reason: Option<String>,
881    pub command_status: Option<String>,
882    pub raw: Value,
883}
884
885/// A stable rejection returned by instance- or selected-run lifecycle commands.
886#[derive(Clone, Debug, Error)]
887#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
888pub struct WorkflowCommandRejection {
889    pub command: WorkflowCommandKind,
890    pub status: u16,
891    pub reason: String,
892    pub message: String,
893    pub workflow_id: String,
894    pub run_id: Option<String>,
895    pub target_scope: Option<String>,
896    pub body: Value,
897}
898
899/// Stable terminal categories returned by [`WorkflowHandle::result`].
900#[derive(Clone, Copy, Debug, PartialEq, Eq)]
901pub enum WorkflowTerminalKind {
902    Failed,
903    Cancelled,
904    Terminated,
905    TimedOut,
906}
907
908/// A typed terminal workflow outcome with durable identity and failure metadata.
909///
910/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
911/// of parsing its display representation. Fields remain `None` when an older
912/// server did not publish that metadata.
913#[derive(Clone, Debug, Error)]
914#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
915pub struct WorkflowTerminalOutcome {
916    pub kind: WorkflowTerminalKind,
917    pub workflow_id: String,
918    pub run_id: Option<String>,
919    pub reason: String,
920    pub failure_category: Option<String>,
921    pub failure_id: Option<String>,
922    pub exception_type: Option<String>,
923    pub exception_class: Option<String>,
924    pub non_retryable: Option<bool>,
925    pub message: Option<String>,
926    pub exception: Option<Value>,
927    pub raw: Value,
928}
929
930/// A worker-side activity settlement or heartbeat rejected by durable state.
931#[derive(Clone, Debug, Error)]
932#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
933pub struct ActivityTaskRejection {
934    pub operation: String,
935    pub status: u16,
936    pub reason: String,
937    pub task_id: String,
938    pub activity_attempt_id: String,
939    pub cancel_requested: bool,
940    pub can_continue: Option<bool>,
941    pub run_closed_reason: Option<String>,
942    pub body: Value,
943}
944
945/// Stable validation categories for [`ActivityOptions`].
946#[derive(Clone, Copy, Debug, PartialEq, Eq)]
947pub enum ActivityOptionsErrorKind {
948    EmptyTaskQueue,
949    EmptyRetryPolicy,
950    InvalidMaxAttempts,
951    BackoffWithoutRetryBudget,
952    TooManyBackoffIntervals,
953    InvalidBackoffCoefficient,
954    BackoffGenerationTooLarge,
955    BackoffOverflow,
956    EmptyNonRetryableErrorType,
957    TimeoutNotPositive,
958    TimeoutOverflow,
959    TimeoutOrder,
960}
961
962/// A machine-readable activity-options validation failure.
963#[derive(Clone, Debug, Error, PartialEq, Eq)]
964#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
965pub struct ActivityOptionsError {
966    pub kind: ActivityOptionsErrorKind,
967    pub field: Option<&'static str>,
968    pub message: String,
969}
970
971impl ActivityOptionsError {
972    fn new(
973        kind: ActivityOptionsErrorKind,
974        field: Option<&'static str>,
975        message: impl Into<String>,
976    ) -> Self {
977        Self {
978            kind,
979            field,
980            message: message.into(),
981        }
982    }
983}
984
985/// Stable terminal categories returned when an awaited activity does not succeed.
986#[derive(Clone, Copy, Debug, PartialEq, Eq)]
987pub enum ActivityFailureKind {
988    Failed,
989    Cancelled,
990    TimedOut,
991}
992
993/// A stable, machine-readable terminal activity failure.
994///
995/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
996/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
997#[derive(Clone, Debug, Error)]
998#[error("activity failed ({reason}): {message}")]
999pub struct ActivityFailure {
1000    pub kind: ActivityFailureKind,
1001    pub reason: String,
1002    pub message: String,
1003    pub activity_execution_id: Option<String>,
1004    pub activity_attempt_id: Option<String>,
1005    pub activity_type: Option<String>,
1006    pub activity_class: Option<String>,
1007    pub attempt_number: Option<u64>,
1008    pub failure_id: Option<String>,
1009    pub failure_category: Option<String>,
1010    pub timeout_kind: Option<String>,
1011    pub non_retryable: bool,
1012    pub exception_type: Option<String>,
1013    pub exception_class: Option<String>,
1014    pub code: Option<Value>,
1015    pub exception: Option<Value>,
1016}
1017
1018/// Stable terminal categories returned when an awaited child does not succeed.
1019#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1020pub enum ChildWorkflowFailureKind {
1021    Failed,
1022    Cancelled,
1023    Terminated,
1024}
1025
1026/// A stable, machine-readable child workflow failure delivered to its parent.
1027///
1028/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
1029/// of parsing the display message. Child and parent identifiers retain the
1030/// relationship recorded in durable history across worker restarts.
1031#[derive(Clone, Debug, Error)]
1032#[error("child workflow failed ({reason}): {message}")]
1033pub struct ChildWorkflowFailure {
1034    pub kind: ChildWorkflowFailureKind,
1035    pub reason: String,
1036    pub message: String,
1037    pub parent_workflow_id: Option<String>,
1038    pub parent_workflow_run_id: Option<String>,
1039    pub child_workflow_id: Option<String>,
1040    pub child_workflow_run_id: Option<String>,
1041    pub child_workflow_type: Option<String>,
1042    pub failure_id: Option<String>,
1043    pub failure_category: Option<String>,
1044    pub exception_type: Option<String>,
1045    pub exception_class: Option<String>,
1046    pub non_retryable: bool,
1047    pub code: Option<Value>,
1048    pub exception: Option<Value>,
1049}
1050
1051/// The identity of one durable workflow execution.
1052#[derive(Clone, Debug, PartialEq, Eq)]
1053pub struct WorkflowIdentity {
1054    pub workflow_id: Option<String>,
1055    pub run_id: Option<String>,
1056}
1057
1058/// A successful child result together with its durable parent-child identity.
1059#[derive(Clone, Debug, PartialEq)]
1060pub struct ChildWorkflowResult {
1061    pub parent: WorkflowIdentity,
1062    pub child: WorkflowIdentity,
1063    pub child_workflow_type: Option<String>,
1064    pub result: Value,
1065}
1066
1067/// Lossless successful child result for fixed Avro Value workflows.
1068#[derive(Clone, Debug, PartialEq)]
1069pub struct ChildWorkflowAvroResult {
1070    pub parent: WorkflowIdentity,
1071    pub child: WorkflowIdentity,
1072    pub child_workflow_type: Option<String>,
1073    pub result: AvroValue,
1074}
1075
1076/// Stable user-facing identity for one member of a durable selection group.
1077#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1078#[serde(untagged)]
1079pub enum SelectionKey {
1080    Index(usize),
1081    Name(String),
1082}
1083
1084impl From<usize> for SelectionKey {
1085    fn from(value: usize) -> Self {
1086        Self::Index(value)
1087    }
1088}
1089
1090impl From<String> for SelectionKey {
1091    fn from(value: String) -> Self {
1092        Self::Name(value)
1093    }
1094}
1095
1096impl From<&str> for SelectionKey {
1097    fn from(value: &str) -> Self {
1098        Self::Name(value.to_string())
1099    }
1100}
1101
1102/// Typed result of explicitly awaiting a cancelled non-winning operation.
1103#[derive(Clone, Debug, Error, PartialEq, Eq)]
1104#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1105pub struct DurableOperationCancelled {
1106    pub selection_group_id: String,
1107    pub member_key: SelectionKey,
1108    pub member_index: usize,
1109    pub operation_kind: String,
1110    pub operation_identity: String,
1111}
1112
1113/// Stable identity for one enclosing deterministic parallel group.
1114///
1115/// The same fields are attached to every ordinary activity, timer, or child
1116/// workflow command in the group. Nested leaves carry an outer-to-inner path;
1117/// no Rust-specific wire command is introduced.
1118#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1119pub struct ParallelGroupMetadata {
1120    pub parallel_group_id: String,
1121    pub parallel_group_kind: String,
1122    pub parallel_group_base_sequence: u64,
1123    pub parallel_group_size: usize,
1124    pub parallel_group_index: usize,
1125    #[serde(default, skip_serializing_if = "Option::is_none")]
1126    pub parallel_group_mode: Option<String>,
1127    #[serde(default, skip_serializing_if = "Option::is_none")]
1128    pub selection_member_key: Option<SelectionKey>,
1129    #[serde(default, skip_serializing_if = "Option::is_none")]
1130    pub selection_member_index: Option<usize>,
1131    #[serde(default, skip_serializing_if = "Option::is_none")]
1132    pub selection_member_base_sequence: Option<u64>,
1133    #[serde(default, skip_serializing_if = "Option::is_none")]
1134    pub selection_member_size: Option<usize>,
1135    #[serde(default, skip_serializing_if = "Option::is_none")]
1136    pub selection_member_kind: Option<String>,
1137}
1138
1139/// One input-ordered result returned by [`WorkflowContext::parallel`].
1140#[derive(Clone, Debug, PartialEq)]
1141pub enum ParallelResult {
1142    Activity(Value),
1143    ChildWorkflow(ChildWorkflowResult),
1144    Timer,
1145    Signal(Vec<Value>),
1146    Condition(ConditionWaitResult),
1147    Group(Vec<ParallelResult>),
1148}
1149
1150/// Lossless fixed-Avro counterpart to [`ParallelResult`].
1151#[derive(Clone, Debug, PartialEq)]
1152pub enum ParallelAvroResult {
1153    Activity(AvroValue),
1154    ChildWorkflow(ChildWorkflowAvroResult),
1155    Timer,
1156    Signal(Vec<AvroValue>),
1157    Condition(ConditionWaitResult),
1158    Group(Vec<ParallelAvroResult>),
1159}
1160
1161impl ParallelAvroResult {
1162    fn into_json_result(self) -> Result<ParallelResult> {
1163        match self {
1164            Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1165            Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1166                parent: result.parent,
1167                child: result.child,
1168                child_workflow_type: result.child_workflow_type,
1169                result: result.result.into_json()?,
1170            })),
1171            Self::Timer => Ok(ParallelResult::Timer),
1172            Self::Signal(values) => Ok(ParallelResult::Signal(
1173                values
1174                    .into_iter()
1175                    .map(AvroValue::into_json)
1176                    .collect::<Result<Vec<_>>>()?,
1177            )),
1178            Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1179            Self::Group(results) => Ok(ParallelResult::Group(
1180                results
1181                    .into_iter()
1182                    .map(Self::into_json_result)
1183                    .collect::<Result<Vec<_>>>()?,
1184            )),
1185        }
1186    }
1187}
1188
1189/// One successful leaf retained when another parallel member failed.
1190#[derive(Clone, Debug, PartialEq)]
1191pub struct ParallelCompletion {
1192    pub member_path: Vec<usize>,
1193    pub result: ParallelResult,
1194}
1195
1196/// A deterministic join failed after some siblings had already completed.
1197///
1198/// `cause` retains the typed activity, child-workflow, cancellation, or codec
1199/// error. `completed` is declaration ordered and contains only durable
1200/// successes observed in the same replay. Late sibling completions can add
1201/// entries on a later replay without changing `member_path` or the selected
1202/// positional failure.
1203#[derive(Debug, Error)]
1204#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1205pub struct ParallelFailure {
1206    pub group_id: String,
1207    pub member_path: Vec<usize>,
1208    pub group_path: Vec<ParallelGroupMetadata>,
1209    pub completed: Vec<ParallelCompletion>,
1210    #[source]
1211    pub cause: Box<Error>,
1212}
1213
1214/// Stable validation error returned before an invalid group emits commands.
1215#[derive(Clone, Debug, Error, PartialEq, Eq)]
1216#[error("invalid deterministic parallel group ({reason}): {message}")]
1217pub struct ParallelGroupError {
1218    pub reason: &'static str,
1219    pub member_path: Vec<usize>,
1220    pub message: String,
1221}
1222
1223/// Cooperative workflow cancellation observed at an author-controlled point.
1224#[derive(Clone, Debug, Error, PartialEq, Eq)]
1225#[error("workflow cancellation was requested")]
1226pub struct WorkflowCancellationRequested;
1227
1228/// A forward saga failure followed by a terminal compensation failure.
1229#[derive(Debug, Error)]
1230#[error(
1231    "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1232)]
1233pub struct SagaCompensationFailure {
1234    pub initiating_failure: Box<Error>,
1235    pub compensation_failure: Box<Error>,
1236    pub compensation_activity_type: String,
1237    pub compensation_registration_order: usize,
1238}
1239
1240/// Server behavior when a parent closes while its child is still open.
1241#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1242pub enum ParentClosePolicy {
1243    #[default]
1244    Abandon,
1245    RequestCancel,
1246    Terminate,
1247}
1248
1249impl ParentClosePolicy {
1250    fn as_str(self) -> &'static str {
1251        match self {
1252            Self::Abandon => "abandon",
1253            Self::RequestCancel => "request_cancel",
1254            Self::Terminate => "terminate",
1255        }
1256    }
1257}
1258
1259/// Durable retry policy for one child workflow invocation.
1260#[derive(Clone, Debug, Default, PartialEq, Eq)]
1261pub struct ChildWorkflowRetryPolicy {
1262    pub max_attempts: Option<u32>,
1263    pub backoff_seconds: Vec<u64>,
1264    pub non_retryable_error_types: Vec<String>,
1265}
1266
1267/// Options recorded with a child-workflow command.
1268///
1269/// The task queue is mandatory so routing is explicit and replay-stable.
1270#[derive(Clone, Debug, PartialEq, Eq)]
1271pub struct ChildWorkflowOptions {
1272    pub task_queue: String,
1273    pub parent_close_policy: ParentClosePolicy,
1274    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1275    pub execution_timeout_seconds: Option<u64>,
1276    pub run_timeout_seconds: Option<u64>,
1277}
1278
1279impl ChildWorkflowOptions {
1280    pub fn new(task_queue: impl Into<String>) -> Self {
1281        Self {
1282            task_queue: task_queue.into(),
1283            parent_close_policy: ParentClosePolicy::Abandon,
1284            retry_policy: None,
1285            execution_timeout_seconds: None,
1286            run_timeout_seconds: None,
1287        }
1288    }
1289
1290    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1291        self.parent_close_policy = policy;
1292        self
1293    }
1294
1295    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1296        self.retry_policy = Some(policy);
1297        self
1298    }
1299
1300    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1301        self.execution_timeout_seconds = Some(seconds);
1302        self
1303    }
1304
1305    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1306        self.run_timeout_seconds = Some(seconds);
1307        self
1308    }
1309}
1310
1311/// Backoff intervals for one durable activity retry policy.
1312#[derive(Clone, Debug, PartialEq, Eq)]
1313pub enum ActivityBackoff {
1314    /// Use these intervals between attempts. The server repeats the final
1315    /// interval if the retry budget contains more attempts than entries.
1316    Explicit(Vec<Duration>),
1317    /// Generate one interval for every retry using integer exponential growth.
1318    Exponential {
1319        initial_interval: Duration,
1320        coefficient: u32,
1321        maximum_interval: Option<Duration>,
1322    },
1323}
1324
1325/// Durable server-side retry policy for one activity execution.
1326#[derive(Clone, Debug, Default, PartialEq, Eq)]
1327pub struct ActivityRetryPolicy {
1328    pub max_attempts: Option<u32>,
1329    pub backoff: Option<ActivityBackoff>,
1330    pub non_retryable_error_types: Vec<String>,
1331}
1332
1333impl ActivityRetryPolicy {
1334    /// Start a policy with a finite attempt budget, including the first attempt.
1335    pub fn new(max_attempts: u32) -> Self {
1336        Self {
1337            max_attempts: Some(max_attempts),
1338            ..Self::default()
1339        }
1340    }
1341
1342    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1343        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1344        self
1345    }
1346
1347    pub fn exponential_backoff(
1348        mut self,
1349        initial_interval: Duration,
1350        coefficient: u32,
1351        maximum_interval: Option<Duration>,
1352    ) -> Self {
1353        self.backoff = Some(ActivityBackoff::Exponential {
1354            initial_interval,
1355            coefficient,
1356            maximum_interval,
1357        });
1358        self
1359    }
1360
1361    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1362        self.non_retryable_error_types.push(error_type.into());
1363        self
1364    }
1365
1366    pub fn non_retryable_error_types(
1367        mut self,
1368        error_types: impl IntoIterator<Item = impl Into<String>>,
1369    ) -> Self {
1370        self.non_retryable_error_types
1371            .extend(error_types.into_iter().map(Into::into));
1372        self
1373    }
1374}
1375
1376/// Options recorded atomically on one deterministic `schedule_activity` command.
1377///
1378/// Durations are rounded up to whole seconds when encoded, so the server never
1379/// receives a shorter timeout or backoff than the caller requested.
1380#[derive(Clone, Debug, Default, PartialEq, Eq)]
1381pub struct ActivityOptions {
1382    pub task_queue: Option<String>,
1383    pub retry_policy: Option<ActivityRetryPolicy>,
1384    pub start_to_close_timeout: Option<Duration>,
1385    pub schedule_to_start_timeout: Option<Duration>,
1386    pub schedule_to_close_timeout: Option<Duration>,
1387    pub heartbeat_timeout: Option<Duration>,
1388}
1389
1390impl ActivityOptions {
1391    pub fn new() -> Self {
1392        Self::default()
1393    }
1394
1395    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1396        self.task_queue = Some(task_queue.into());
1397        self
1398    }
1399
1400    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1401        self.retry_policy = Some(policy);
1402        self
1403    }
1404
1405    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1406        self.start_to_close_timeout = Some(timeout);
1407        self
1408    }
1409
1410    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1411        self.schedule_to_start_timeout = Some(timeout);
1412        self
1413    }
1414
1415    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1416        self.schedule_to_close_timeout = Some(timeout);
1417        self
1418    }
1419
1420    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1421        self.heartbeat_timeout = Some(timeout);
1422        self
1423    }
1424
1425    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1426        if self
1427            .task_queue
1428            .as_deref()
1429            .is_some_and(|queue| queue.trim().is_empty())
1430        {
1431            return Err(ActivityOptionsError::new(
1432                ActivityOptionsErrorKind::EmptyTaskQueue,
1433                Some("task_queue"),
1434                "task_queue must not be empty",
1435            ));
1436        }
1437
1438        for (field, value) in [
1439            ("start_to_close_timeout", self.start_to_close_timeout),
1440            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1441            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1442            ("heartbeat_timeout", self.heartbeat_timeout),
1443        ] {
1444            if value.is_some_and(|value| value.is_zero()) {
1445                return Err(ActivityOptionsError::new(
1446                    ActivityOptionsErrorKind::TimeoutNotPositive,
1447                    Some(field),
1448                    format!("{field} must be positive"),
1449                ));
1450            }
1451        }
1452
1453        validate_timeout_order(
1454            "heartbeat_timeout",
1455            self.heartbeat_timeout,
1456            "start_to_close_timeout",
1457            self.start_to_close_timeout,
1458        )?;
1459        validate_timeout_order(
1460            "start_to_close_timeout",
1461            self.start_to_close_timeout,
1462            "schedule_to_close_timeout",
1463            self.schedule_to_close_timeout,
1464        )?;
1465        validate_timeout_order(
1466            "schedule_to_start_timeout",
1467            self.schedule_to_start_timeout,
1468            "schedule_to_close_timeout",
1469            self.schedule_to_close_timeout,
1470        )?;
1471
1472        Ok(ValidatedActivityOptions {
1473            task_queue: self.task_queue.clone(),
1474            retry_policy: self
1475                .retry_policy
1476                .as_ref()
1477                .map(validate_activity_retry_policy)
1478                .transpose()?,
1479            start_to_close_timeout: timeout_seconds(
1480                "start_to_close_timeout",
1481                self.start_to_close_timeout,
1482            )?,
1483            schedule_to_start_timeout: timeout_seconds(
1484                "schedule_to_start_timeout",
1485                self.schedule_to_start_timeout,
1486            )?,
1487            schedule_to_close_timeout: timeout_seconds(
1488                "schedule_to_close_timeout",
1489                self.schedule_to_close_timeout,
1490            )?,
1491            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1492        })
1493    }
1494}
1495
1496/// A deferred durable leaf or nested group for [`WorkflowContext::parallel`].
1497///
1498/// Constructors capture arguments but perform no I/O. The join validates the
1499/// complete tree, attaches the existing parallel-group metadata to every
1500/// ordinary command, schedules all leaves, and then suspends.
1501pub enum ParallelOperation {
1502    Activity {
1503        activity_type: String,
1504        options: ActivityOptions,
1505        arguments: Result<AvroValue>,
1506    },
1507    ChildWorkflow {
1508        workflow_type: String,
1509        options: ChildWorkflowOptions,
1510        arguments: Result<AvroValue>,
1511    },
1512    Timer(Duration),
1513    Signal(String),
1514    Condition {
1515        options: ConditionWaitOptions,
1516        predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1517    },
1518    Group(Vec<ParallelOperation>),
1519}
1520
1521impl ParallelOperation {
1522    pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1523        Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1524    }
1525
1526    pub fn activity_with_options<T: Serialize>(
1527        activity_type: impl Into<String>,
1528        options: ActivityOptions,
1529        args: T,
1530    ) -> Self {
1531        Self::Activity {
1532            activity_type: activity_type.into(),
1533            options,
1534            arguments: AvroValue::from_serialize(&args),
1535        }
1536    }
1537
1538    pub fn child_workflow<T: Serialize>(
1539        workflow_type: impl Into<String>,
1540        options: ChildWorkflowOptions,
1541        args: T,
1542    ) -> Self {
1543        Self::ChildWorkflow {
1544            workflow_type: workflow_type.into(),
1545            options,
1546            arguments: AvroValue::from_serialize(&args),
1547        }
1548    }
1549
1550    pub fn timer(duration: Duration) -> Self {
1551        Self::Timer(duration)
1552    }
1553
1554    pub fn signal(signal_name: impl Into<String>) -> Self {
1555        Self::Signal(signal_name.into())
1556    }
1557
1558    pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1559    where
1560        F: Fn() -> Result<bool> + Send + 'static,
1561    {
1562        Self::Condition {
1563            options,
1564            predicate: Box::new(predicate),
1565        }
1566    }
1567
1568    pub fn group(operations: Vec<ParallelOperation>) -> Self {
1569        Self::Group(operations)
1570    }
1571}
1572
1573#[derive(Clone, Debug)]
1574struct ValidatedActivityOptions {
1575    task_queue: Option<String>,
1576    retry_policy: Option<Value>,
1577    start_to_close_timeout: Option<u64>,
1578    schedule_to_start_timeout: Option<u64>,
1579    schedule_to_close_timeout: Option<u64>,
1580    heartbeat_timeout: Option<u64>,
1581}
1582
1583fn validate_timeout_order(
1584    smaller_name: &'static str,
1585    smaller: Option<Duration>,
1586    larger_name: &'static str,
1587    larger: Option<Duration>,
1588) -> std::result::Result<(), ActivityOptionsError> {
1589    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1590        return Err(ActivityOptionsError::new(
1591            ActivityOptionsErrorKind::TimeoutOrder,
1592            Some(smaller_name),
1593            format!("{smaller_name} must be <= {larger_name}"),
1594        ));
1595    }
1596    Ok(())
1597}
1598
1599fn timeout_seconds(
1600    field: &'static str,
1601    value: Option<Duration>,
1602) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1603    value
1604        .map(|value| {
1605            activity_protocol_seconds(value).ok_or_else(|| {
1606                ActivityOptionsError::new(
1607                    ActivityOptionsErrorKind::TimeoutOverflow,
1608                    Some(field),
1609                    format!("{field} is too large for the worker protocol"),
1610                )
1611            })
1612        })
1613        .transpose()
1614}
1615
1616fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1617    value
1618        .as_secs()
1619        .checked_add(u64::from(value.subsec_nanos() > 0))
1620}
1621
1622fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1623    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1624}
1625
1626fn validate_activity_retry_policy(
1627    policy: &ActivityRetryPolicy,
1628) -> std::result::Result<Value, ActivityOptionsError> {
1629    if policy.max_attempts.is_none()
1630        && policy.backoff.is_none()
1631        && policy.non_retryable_error_types.is_empty()
1632    {
1633        return Err(ActivityOptionsError::new(
1634            ActivityOptionsErrorKind::EmptyRetryPolicy,
1635            Some("retry_policy"),
1636            "retry_policy must configure at least one field",
1637        ));
1638    }
1639    if policy.max_attempts == Some(0) {
1640        return Err(ActivityOptionsError::new(
1641            ActivityOptionsErrorKind::InvalidMaxAttempts,
1642            Some("retry_policy.max_attempts"),
1643            "max_attempts must be >= 1",
1644        ));
1645    }
1646    if policy
1647        .non_retryable_error_types
1648        .iter()
1649        .any(|error_type| error_type.trim().is_empty())
1650    {
1651        return Err(ActivityOptionsError::new(
1652            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1653            Some("retry_policy.non_retryable_error_types"),
1654            "non_retryable_error_types must not contain empty values",
1655        ));
1656    }
1657
1658    let backoff_seconds = match &policy.backoff {
1659        None => None,
1660        Some(backoff) => {
1661            let max_attempts = policy.max_attempts.ok_or_else(|| {
1662                ActivityOptionsError::new(
1663                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1664                    Some("retry_policy.backoff"),
1665                    "backoff requires max_attempts",
1666                )
1667            })?;
1668            let retry_count = max_attempts.saturating_sub(1) as usize;
1669            let intervals = match backoff {
1670                ActivityBackoff::Explicit(intervals) => {
1671                    if intervals.len() > retry_count {
1672                        return Err(ActivityOptionsError::new(
1673                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
1674                            Some("retry_policy.backoff"),
1675                            "backoff interval count must not exceed max_attempts - 1",
1676                        ));
1677                    }
1678                    intervals.clone()
1679                }
1680                ActivityBackoff::Exponential {
1681                    initial_interval,
1682                    coefficient,
1683                    maximum_interval,
1684                } => {
1685                    if *coefficient < 1 {
1686                        return Err(ActivityOptionsError::new(
1687                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1688                            Some("retry_policy.backoff.coefficient"),
1689                            "backoff coefficient must be >= 1",
1690                        ));
1691                    }
1692                    if retry_count > 10_000 {
1693                        return Err(ActivityOptionsError::new(
1694                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1695                            Some("retry_policy.max_attempts"),
1696                            "generated backoff supports at most 10000 retry intervals",
1697                        ));
1698                    }
1699                    let mut current = *initial_interval;
1700                    let mut intervals = Vec::with_capacity(retry_count);
1701                    for _ in 0..retry_count {
1702                        let interval = maximum_interval
1703                            .map(|maximum| current.min(maximum))
1704                            .unwrap_or(current);
1705                        intervals.push(interval);
1706                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
1707                            break;
1708                        }
1709                        current = current.checked_mul(*coefficient).ok_or_else(|| {
1710                            ActivityOptionsError::new(
1711                                ActivityOptionsErrorKind::BackoffOverflow,
1712                                Some("retry_policy.backoff"),
1713                                "generated backoff interval overflowed",
1714                            )
1715                        })?;
1716                    }
1717                    intervals
1718                }
1719            };
1720            Some(
1721                intervals
1722                    .into_iter()
1723                    .map(|interval| {
1724                        activity_protocol_seconds(interval).ok_or_else(|| {
1725                            ActivityOptionsError::new(
1726                                ActivityOptionsErrorKind::BackoffOverflow,
1727                                Some("retry_policy.backoff"),
1728                                "backoff interval is too large for the worker protocol",
1729                            )
1730                        })
1731                    })
1732                    .collect::<std::result::Result<Vec<_>, _>>()?,
1733            )
1734        }
1735    };
1736
1737    let mut encoded = serde_json::Map::new();
1738    if let Some(max_attempts) = policy.max_attempts {
1739        encoded.insert("max_attempts".to_string(), json!(max_attempts));
1740    }
1741    if let Some(backoff_seconds) = backoff_seconds {
1742        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1743    }
1744    if !policy.non_retryable_error_types.is_empty() {
1745        let mut canonical_error_types = Vec::new();
1746        for error_type in policy
1747            .non_retryable_error_types
1748            .iter()
1749            .map(|error_type| error_type.trim())
1750        {
1751            if !canonical_error_types.contains(&error_type) {
1752                canonical_error_types.push(error_type);
1753            }
1754        }
1755        encoded.insert(
1756            "non_retryable_error_types".to_string(),
1757            json!(canonical_error_types),
1758        );
1759    }
1760    Ok(Value::Object(encoded))
1761}
1762
1763/// A stable, machine-readable failure raised when workflow code no longer
1764/// reconstructs the durable command stream recorded in history.
1765#[derive(Clone, Debug, Error)]
1766#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1767pub struct ReplayFailure {
1768    pub reason: String,
1769    pub sequence: Option<u64>,
1770    pub expected: Option<String>,
1771    pub actual: Option<String>,
1772    pub message: String,
1773}
1774
1775impl ReplayFailure {
1776    fn new(
1777        reason: impl Into<String>,
1778        sequence: Option<u64>,
1779        expected: Option<String>,
1780        actual: Option<String>,
1781        message: impl Into<String>,
1782    ) -> Self {
1783        Self {
1784            reason: reason.into(),
1785            sequence,
1786            expected,
1787            actual,
1788            message: message.into(),
1789        }
1790    }
1791}
1792
1793/// A stable, machine-readable workflow query or query-task settlement failure.
1794#[derive(Clone, Debug, Error)]
1795#[error("query failed ({reason}, HTTP {status}): {message}")]
1796pub struct QueryFailure {
1797    pub status: u16,
1798    pub reason: String,
1799    pub message: String,
1800    pub body: Value,
1801}
1802
1803/// A stable failure returned when a server rejects an SDK protocol version.
1804#[derive(Clone, Debug, Error)]
1805#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1806pub struct ProtocolFailure {
1807    pub status: u16,
1808    pub reason: String,
1809    pub message: String,
1810    pub supported_version: Option<String>,
1811    pub requested_version: Option<String>,
1812    pub body: Value,
1813}
1814
1815#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1816pub struct PayloadEnvelope {
1817    pub codec: String,
1818    pub blob: String,
1819}
1820
1821impl PayloadEnvelope {
1822    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1823        encode_payload(value, DEFAULT_CODEC)
1824    }
1825
1826    /// Encode an explicit typed value, including the bytes branch that JSON
1827    /// serialization cannot represent.
1828    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1829        encode_avro_value(value)
1830    }
1831}
1832
1833/// Native adapter for the fixed language-neutral Avro Value schema.
1834#[derive(Clone, Debug)]
1835pub enum AvroValue {
1836    Null,
1837    Boolean(bool),
1838    Long(i64),
1839    Double(f64),
1840    Bytes(Vec<u8>),
1841    String(String),
1842    Array(Vec<AvroValue>),
1843    Map(BTreeMap<String, AvroValue>),
1844}
1845
1846impl PartialEq for AvroValue {
1847    fn eq(&self, other: &Self) -> bool {
1848        match (self, other) {
1849            (Self::Null, Self::Null) => true,
1850            (Self::Boolean(left), Self::Boolean(right)) => left == right,
1851            (Self::Long(left), Self::Long(right)) => left == right,
1852            (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1853            (Self::Bytes(left), Self::Bytes(right)) => left == right,
1854            (Self::String(left), Self::String(right)) => left == right,
1855            (Self::Array(left), Self::Array(right)) => left == right,
1856            (Self::Map(left), Self::Map(right)) => left == right,
1857            _ => false,
1858        }
1859    }
1860}
1861
1862impl AvroValue {
1863    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1864        Self::from_serde_value(
1865            serde_value::to_value(value).map_err(|error| {
1866                Error::Codec(format!("could not adapt value for Avro: {error}"))
1867            })?,
1868        )
1869    }
1870
1871    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1872        use serde_value::Value as SerdeValue;
1873
1874        match value {
1875            SerdeValue::Unit => Ok(Self::Null),
1876            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1877            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1878            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1879            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1880            SerdeValue::I64(value) => Ok(Self::Long(value)),
1881            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1882            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1883            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1884            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1885                Error::Codec(
1886                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1887                        .to_string(),
1888                )
1889            }),
1890            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1891            SerdeValue::F64(value) => Self::finite_double(value),
1892            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1893            SerdeValue::String(value) => Ok(Self::String(value)),
1894            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1895            SerdeValue::Option(None) => Ok(Self::Null),
1896            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1897                Self::from_serde_value(*value)
1898            }
1899            SerdeValue::Seq(values) => values
1900                .into_iter()
1901                .map(Self::from_serde_value)
1902                .collect::<Result<Vec<_>>>()
1903                .map(Self::Array),
1904            SerdeValue::Map(values) => values
1905                .into_iter()
1906                .map(|(key, value)| {
1907                    let SerdeValue::String(key) = key else {
1908                        return Err(Error::Codec(
1909                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1910                        ));
1911                    };
1912
1913                    Ok((key, Self::from_serde_value(value)?))
1914                })
1915                .collect::<Result<BTreeMap<_, _>>>()
1916                .map(Self::Map),
1917        }
1918    }
1919
1920    fn finite_double(value: f64) -> Result<Self> {
1921        if !value.is_finite() {
1922            return Err(Error::Codec(
1923                "non_finite_float: Avro Value doubles must be finite".to_string(),
1924            ));
1925        }
1926
1927        Ok(Self::Double(value))
1928    }
1929
1930    fn into_json(self) -> Result<Value> {
1931        match self {
1932            Self::Null => Ok(Value::Null),
1933            Self::Boolean(value) => Ok(Value::Bool(value)),
1934            Self::Long(value) => Ok(Value::Number(value.into())),
1935            Self::Double(value) => serde_json::Number::from_f64(value)
1936                .map(Value::Number)
1937                .ok_or_else(|| {
1938                    Error::Codec(
1939                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1940                    )
1941                }),
1942            Self::Bytes(value) => Ok(json!({
1943                "$type": "bytes",
1944                "base64": BASE64.encode(value),
1945            })),
1946            Self::String(value) => Ok(Value::String(value)),
1947            Self::Array(values) => values
1948                .into_iter()
1949                .map(Self::into_json)
1950                .collect::<Result<Vec<_>>>()
1951                .map(Value::Array),
1952            Self::Map(values) => values
1953                .into_iter()
1954                .map(|(key, value)| Ok((key, value.into_json()?)))
1955                .collect::<Result<serde_json::Map<_, _>>>()
1956                .map(Value::Object),
1957        }
1958    }
1959
1960    fn into_serde_value(self) -> serde_value::Value {
1961        use serde_value::Value as SerdeValue;
1962
1963        match self {
1964            Self::Null => SerdeValue::Unit,
1965            Self::Boolean(value) => SerdeValue::Bool(value),
1966            Self::Long(value) => SerdeValue::I64(value),
1967            Self::Double(value) => SerdeValue::F64(value),
1968            Self::Bytes(value) => SerdeValue::Bytes(value),
1969            Self::String(value) => SerdeValue::String(value),
1970            Self::Array(values) => {
1971                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1972            }
1973            Self::Map(values) => SerdeValue::Map(
1974                values
1975                    .into_iter()
1976                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1977                    .collect(),
1978            ),
1979        }
1980    }
1981
1982    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1983        self.into_serde_value().deserialize_into().map_err(|error| {
1984            Error::Codec(format!(
1985                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1986            ))
1987        })
1988    }
1989}
1990
1991impl Serialize for AvroValue {
1992    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1993    where
1994        S: Serializer,
1995    {
1996        match self {
1997            Self::Null => serializer.serialize_unit(),
1998            Self::Boolean(value) => serializer.serialize_bool(*value),
1999            Self::Long(value) => serializer.serialize_i64(*value),
2000            Self::Double(value) => serializer.serialize_f64(*value),
2001            Self::Bytes(value) => serializer.serialize_bytes(value),
2002            Self::String(value) => serializer.serialize_str(value),
2003            Self::Array(values) => {
2004                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2005                for value in values {
2006                    sequence.serialize_element(value)?;
2007                }
2008                sequence.end()
2009            }
2010            Self::Map(values) => {
2011                let mut map = serializer.serialize_map(Some(values.len()))?;
2012                for (key, value) in values {
2013                    map.serialize_entry(key, value)?;
2014                }
2015                map.end()
2016            }
2017        }
2018    }
2019}
2020
2021pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2022    let datum = avro_value_to_datum(value)?;
2023    let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2024        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2025    let mut bytes = Vec::with_capacity(datum.len() + 10);
2026    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2027    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2028    bytes.extend_from_slice(&datum);
2029    Ok(PayloadEnvelope {
2030        codec: DEFAULT_CODEC.to_string(),
2031        blob: BASE64.encode(bytes),
2032    })
2033}
2034
2035pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2036    if envelope.codec != DEFAULT_CODEC {
2037        return Err(unsupported_payload_codec(&envelope.codec));
2038    }
2039    decode_avro_value_blob(&envelope.blob)
2040}
2041
2042pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2043    let blob = match codec {
2044        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2045        other => return Err(unsupported_payload_codec(other)),
2046    };
2047
2048    Ok(PayloadEnvelope {
2049        codec: codec.to_string(),
2050        blob,
2051    })
2052}
2053
2054pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2055    match envelope.codec.as_str() {
2056        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2057        other => Err(unsupported_payload_codec(other)),
2058    }
2059}
2060
2061fn handler_type_error<T>(
2062    handler_kind: HandlerKind,
2063    handler_name: &str,
2064    value_kind: HandlerValueKind,
2065    message: impl Into<String>,
2066) -> Error {
2067    Error::HandlerType {
2068        handler_kind,
2069        handler_name: handler_name.to_string(),
2070        value_kind,
2071        rust_type: type_name::<T>(),
2072        message: message.into(),
2073    }
2074}
2075
2076fn decode_handler_input<T: DeserializeOwned>(
2077    arguments: AvroValue,
2078    handler_kind: HandlerKind,
2079    handler_name: &str,
2080) -> Result<T> {
2081    let argument = match arguments {
2082        AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2083            arguments.pop().expect("one typed handler argument")
2084        }
2085        AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2086        AvroValue::Array(arguments) => {
2087            return Err(handler_type_error::<T>(
2088                handler_kind,
2089                handler_name,
2090                HandlerValueKind::Input,
2091                format!(
2092                    "typed handlers accept one request value, but the task carried {} arguments",
2093                    arguments.len()
2094                ),
2095            ));
2096        }
2097        argument => argument,
2098    };
2099
2100    argument.deserialize().map_err(|error| {
2101        handler_type_error::<T>(
2102            handler_kind,
2103            handler_name,
2104            HandlerValueKind::Input,
2105            error.to_string(),
2106        )
2107    })
2108}
2109
2110fn encode_handler_result<T: Serialize>(
2111    result: &T,
2112    handler_kind: HandlerKind,
2113    handler_name: &str,
2114) -> Result<AvroValue> {
2115    AvroValue::from_serialize(result).map_err(|error| {
2116        handler_type_error::<T>(
2117            handler_kind,
2118            handler_name,
2119            HandlerValueKind::Result,
2120            error.to_string(),
2121        )
2122    })
2123}
2124
2125fn decode_handler_result<T: DeserializeOwned>(
2126    result: AvroValue,
2127    handler_kind: HandlerKind,
2128    handler_name: &str,
2129) -> Result<T> {
2130    result.deserialize().map_err(|error| {
2131        handler_type_error::<T>(
2132            handler_kind,
2133            handler_name,
2134            HandlerValueKind::Result,
2135            error.to_string(),
2136        )
2137    })
2138}
2139
2140#[cfg(test)]
2141fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2142    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2143}
2144
2145fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2146    validate_payload_codec(fallback_codec)?;
2147
2148    if value.is_null() {
2149        return Ok(Value::Null);
2150    }
2151
2152    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2153        return decode_blob(blob, codec);
2154    }
2155
2156    if let Some(blob) = value.as_str() {
2157        return decode_blob(blob, fallback_codec);
2158    }
2159
2160    Err(untagged_payload_value())
2161}
2162
2163fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2164    let envelope = match codec {
2165        DEFAULT_CODEC => encode_avro_value(value)?,
2166        other => return Err(unsupported_payload_codec(other)),
2167    };
2168    Ok(serde_json::to_value(envelope)?)
2169}
2170
2171fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2172    validate_payload_codec(fallback_codec)?;
2173
2174    if value.is_null() {
2175        return Ok(AvroValue::Null);
2176    }
2177
2178    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2179        validate_payload_codec(codec)?;
2180        return decode_avro_value_blob(blob);
2181    }
2182
2183    if let Some(blob) = value.as_str() {
2184        return match fallback_codec {
2185            DEFAULT_CODEC => decode_avro_value_blob(blob),
2186            other => Err(unsupported_payload_codec(other)),
2187        };
2188    }
2189
2190    Err(untagged_payload_value())
2191}
2192
2193fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2194    match value {
2195        AvroValue::Null => AvroValue::Array(Vec::new()),
2196        AvroValue::Array(_) => value,
2197        other => AvroValue::Array(vec![other]),
2198    }
2199}
2200
2201fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2202    match codec {
2203        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2204        other => Err(unsupported_payload_codec(other)),
2205    }
2206}
2207
2208fn validate_payload_codec(codec: &str) -> Result<()> {
2209    match codec {
2210        DEFAULT_CODEC => Ok(()),
2211        MISSING_TASK_PAYLOAD_CODEC => {
2212            Err(invalid_task_payload_codec("task payload_codec is missing"))
2213        }
2214        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2215        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2216            "task payload_codec must be a string",
2217        )),
2218        other => Err(unsupported_payload_codec(other)),
2219    }
2220}
2221
2222fn invalid_task_payload_codec(reason: &str) -> Error {
2223    Error::Codec(format!(
2224        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2225    ))
2226}
2227
2228fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2229    let Some(object) = value.as_object() else {
2230        return Ok(None);
2231    };
2232    if !object.contains_key("codec") && !object.contains_key("blob") {
2233        return Ok(None);
2234    }
2235
2236    let codec = object
2237        .get("codec")
2238        .and_then(Value::as_str)
2239        .ok_or_else(invalid_payload_envelope)?;
2240    validate_payload_codec(codec)?;
2241    let blob = object
2242        .get("blob")
2243        .and_then(Value::as_str)
2244        .ok_or_else(invalid_payload_envelope)?;
2245    Ok(Some((codec, blob)))
2246}
2247
2248fn invalid_payload_envelope() -> Error {
2249    Error::Codec(
2250        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2251            .to_string(),
2252    )
2253}
2254
2255fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2256    for command in commands {
2257        let Some(command) = command.as_object() else {
2258            continue;
2259        };
2260        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2261            continue;
2262        };
2263        let Some(payload_field) = workflow_command_payload_field(command_type) else {
2264            continue;
2265        };
2266
2267        if let Some(codec) = command.get("payload_codec") {
2268            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2269            validate_payload_codec(codec)?;
2270        }
2271
2272        let payload = command
2273            .get(payload_field)
2274            .ok_or_else(invalid_payload_envelope)?;
2275        validate_outbound_payload_envelope(payload)?;
2276    }
2277    Ok(())
2278}
2279
2280fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2281    if commands.iter().any(|command| {
2282        command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2283            && command
2284                .get("condition_wait_occurrence_id")
2285                .and_then(Value::as_str)
2286                .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2287    }) {
2288        CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2289    } else if commands.iter().any(|command| {
2290        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2291            && command.get("attribute_types").is_some()
2292    }) {
2293        TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2294    } else if commands
2295        .iter()
2296        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2297    {
2298        MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2299    } else if commands
2300        .iter()
2301        .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2302    {
2303        CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2304    } else if commands.iter().any(|command| {
2305        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2306    }) {
2307        SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2308    } else {
2309        WORKER_PROTOCOL_VERSION
2310    }
2311}
2312
2313fn workflow_completion_protocol_version_with_message_streams(
2314    commands: &[Value],
2315    has_message_stream_metadata: bool,
2316) -> &'static str {
2317    let command_protocol = workflow_completion_protocol_version(commands);
2318    if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2319        MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2320    } else {
2321        command_protocol
2322    }
2323}
2324
2325fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2326    match command_type {
2327        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2328        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2329        "start_service_operation" => Some("request_payload"),
2330        "upsert_memo" => Some("entries"),
2331        _ => None,
2332    }
2333}
2334
2335fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2336    let Some((codec, blob)) = payload_envelope_parts(value)? else {
2337        return Err(untagged_payload_value());
2338    };
2339    validate_payload_codec(codec)?;
2340    decode_avro_value_blob(blob)?;
2341    Ok(())
2342}
2343
2344fn unsupported_payload_codec(codec: &str) -> Error {
2345    Error::Codec(format!(
2346        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2347    ))
2348}
2349
2350fn untagged_payload_value() -> Error {
2351    Error::Codec(
2352        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2353            .to_string(),
2354    )
2355}
2356
2357fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2358    let bytes = BASE64.decode(blob).map_err(|err| {
2359        Error::Codec(format!(
2360            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2361        ))
2362    })?;
2363
2364    if serde_json::from_slice::<Value>(&bytes).is_ok() {
2365        return Err(unsupported_payload_codec("json"));
2366    }
2367
2368    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2369        return Err(Error::Codec(
2370            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2371        ));
2372    }
2373
2374    let fingerprint: [u8; 8] = bytes[2..10]
2375        .try_into()
2376        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2377    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2378        return Err(Error::Codec(format!(
2379            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2380            fingerprint
2381                .iter()
2382                .map(|byte| format!("{byte:02x}"))
2383                .collect::<String>()
2384        )));
2385    }
2386
2387    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2388    // The current fingerprint selects the current immutable schema, so reader
2389    // resolution would only re-walk the same recursive union. Future retained
2390    // writer fingerprints supply a distinct reader schema in this branch.
2391    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2392    if datum_reader.truncated {
2393        return Err(Error::Codec(
2394            "invalid_payload_framing: truncated Avro Value datum".to_string(),
2395        ));
2396    }
2397    let datum = datum.map_err(|err| {
2398        Error::Codec(format!(
2399            "invalid_payload_framing: malformed Avro Value datum: {err}"
2400        ))
2401    })?;
2402    if datum_reader.remaining() != 0 {
2403        return Err(Error::Codec(format!(
2404            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2405            datum_reader.remaining()
2406        )));
2407    }
2408    avro_value_from_datum(datum)
2409}
2410
2411struct StrictAvroDatumReader<'a> {
2412    bytes: &'a [u8],
2413    offset: usize,
2414    truncated: bool,
2415}
2416
2417impl<'a> StrictAvroDatumReader<'a> {
2418    fn new(bytes: &'a [u8]) -> Self {
2419        Self {
2420            bytes,
2421            offset: 0,
2422            truncated: false,
2423        }
2424    }
2425
2426    fn remaining(&self) -> usize {
2427        self.bytes.len() - self.offset
2428    }
2429}
2430
2431impl Read for StrictAvroDatumReader<'_> {
2432    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2433        let count = buffer.len().min(self.remaining());
2434        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2435        self.offset += count;
2436        if count < buffer.len() {
2437            self.truncated = true;
2438        }
2439
2440        Ok(count)
2441    }
2442}
2443
2444fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2445    let branch = match value {
2446        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2447        AvroValue::Boolean(value) => AvroDatum::Union(
2448            1,
2449            Box::new(AvroDatum::Record(vec![(
2450                "boolean".to_string(),
2451                AvroDatum::Boolean(*value),
2452            )])),
2453        ),
2454        AvroValue::Long(value) => AvroDatum::Union(
2455            2,
2456            Box::new(AvroDatum::Record(vec![(
2457                "long".to_string(),
2458                AvroDatum::Long(*value),
2459            )])),
2460        ),
2461        AvroValue::Double(value) => {
2462            if !value.is_finite() {
2463                return Err(Error::Codec(
2464                    "non_finite_float: Avro Value doubles must be finite".to_string(),
2465                ));
2466            }
2467            AvroDatum::Union(
2468                3,
2469                Box::new(AvroDatum::Record(vec![(
2470                    "double".to_string(),
2471                    AvroDatum::Double(*value),
2472                )])),
2473            )
2474        }
2475        AvroValue::Bytes(value) => AvroDatum::Union(
2476            4,
2477            Box::new(AvroDatum::Record(vec![(
2478                "bytes".to_string(),
2479                AvroDatum::Bytes(value.clone()),
2480            )])),
2481        ),
2482        AvroValue::String(value) => AvroDatum::Union(
2483            5,
2484            Box::new(AvroDatum::Record(vec![(
2485                "string".to_string(),
2486                AvroDatum::String(value.clone()),
2487            )])),
2488        ),
2489        AvroValue::Array(values) => AvroDatum::Union(
2490            6,
2491            Box::new(AvroDatum::Record(vec![(
2492                "items".to_string(),
2493                AvroDatum::Array(
2494                    values
2495                        .iter()
2496                        .map(avro_value_to_datum)
2497                        .collect::<Result<Vec<_>>>()?,
2498                ),
2499            )])),
2500        ),
2501        AvroValue::Map(values) => AvroDatum::Union(
2502            7,
2503            Box::new(AvroDatum::Record(vec![(
2504                "entries".to_string(),
2505                AvroDatum::Array(
2506                    values
2507                        .iter()
2508                        .map(|(key, value)| {
2509                            Ok(AvroDatum::Record(vec![
2510                                ("key".to_string(), AvroDatum::String(key.clone())),
2511                                ("value".to_string(), avro_value_to_datum(value)?),
2512                            ]))
2513                        })
2514                        .collect::<Result<Vec<_>>>()?,
2515                ),
2516            )])),
2517        ),
2518    };
2519    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2520}
2521
2522fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2523    let AvroDatum::Record(mut outer) = datum else {
2524        return Err(Error::Codec(
2525            "invalid_payload_framing: datum is not a Value record".to_string(),
2526        ));
2527    };
2528    let (_, branch) = outer
2529        .pop()
2530        .filter(|(name, _)| name == "value")
2531        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2532    let AvroDatum::Union(_, branch) = branch else {
2533        return Err(Error::Codec(
2534            "invalid_payload_framing: invalid Value union".to_string(),
2535        ));
2536    };
2537    match *branch {
2538        AvroDatum::Null => Ok(AvroValue::Null),
2539        AvroDatum::Record(mut fields) => {
2540            let (name, value) = fields.pop().ok_or_else(|| {
2541                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2542            })?;
2543            match (name.as_str(), value) {
2544                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2545                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2546                ("double", AvroDatum::Double(value)) if value.is_finite() => {
2547                    Ok(AvroValue::Double(value))
2548                }
2549                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2550                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2551                ("items", AvroDatum::Array(values)) => values
2552                    .into_iter()
2553                    .map(avro_value_from_datum)
2554                    .collect::<Result<Vec<_>>>()
2555                    .map(AvroValue::Array),
2556                ("entries", AvroDatum::Map(values)) => values
2557                    .into_iter()
2558                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2559                    .collect::<Result<BTreeMap<_, _>>>()
2560                    .map(AvroValue::Map),
2561                _ => Err(Error::Codec(
2562                    "invalid_payload_framing: unknown Value branch".to_string(),
2563                )),
2564            }
2565        }
2566        _ => Err(Error::Codec(
2567            "invalid_payload_framing: invalid Value branch".to_string(),
2568        )),
2569    }
2570}
2571
2572fn avro_value_schema() -> Result<&'static Schema> {
2573    match AVRO_VALUE_SCHEMA.get_or_init(|| {
2574        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2575            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2576    }) {
2577        Ok(schema) => Ok(schema),
2578        Err(message) => Err(Error::Codec(message.clone())),
2579    }
2580}
2581
2582fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2583    match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2584        // Apache Avro's Value::Map uses a randomized HashMap. Arrays and maps
2585        // have the same block representation when each ordered array record
2586        // contains the map key followed by its value, so this encoding-only
2587        // adaptation lets the official encoder retain BTreeMap wire order.
2588        let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2589            .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2590        let entries_schema = schema
2591            .pointer_mut("/fields/0/type/7/fields/0/type")
2592            .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2593        if *entries_schema != json!({"type": "map", "values": "Value"}) {
2594            return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2595        }
2596        *entries_schema = json!({
2597            "type": "array",
2598            "items": {
2599                "type": "record",
2600                "name": "MapEntry",
2601                "fields": [
2602                    {"name": "key", "type": "string"},
2603                    {"name": "value", "type": "Value"}
2604                ]
2605            }
2606        });
2607        Schema::parse_str(&schema.to_string())
2608            .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2609    }) {
2610        Ok(schema) => Ok(schema),
2611        Err(message) => Err(Error::Codec(message.clone())),
2612    }
2613}
2614
2615#[derive(Clone, Debug)]
2616pub struct Client {
2617    http: reqwest::Client,
2618    base_url: String,
2619    token: Option<String>,
2620    control_token: Option<String>,
2621    worker_token: Option<String>,
2622    namespace: String,
2623    max_external_payload_bytes: usize,
2624    worker_storage_admission: Option<WorkerStorageAdmission>,
2625}
2626
2627impl Client {
2628    pub fn new(base_url: impl Into<String>) -> Result<Self> {
2629        Self::builder(base_url).build()
2630    }
2631
2632    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2633        ClientBuilder {
2634            base_url: base_url.into(),
2635            token: None,
2636            control_token: None,
2637            worker_token: None,
2638            namespace: "default".to_string(),
2639            timeout: Duration::from_secs(60),
2640            max_external_payload_bytes: 64 * 1024 * 1024,
2641        }
2642    }
2643
2644    pub async fn health(&self) -> Result<Value> {
2645        self.request_json(
2646            reqwest::Method::GET,
2647            "/health",
2648            RequestProtocol::ControlPlane,
2649            Option::<&Value>::None,
2650        )
2651        .await
2652    }
2653
2654    pub async fn cluster_info(&self) -> Result<Value> {
2655        self.request_json(
2656            reqwest::Method::GET,
2657            "/cluster/info",
2658            RequestProtocol::ControlPlane,
2659            Option::<&Value>::None,
2660        )
2661        .await
2662    }
2663
2664    pub async fn start_workflow<T: Serialize>(
2665        &self,
2666        workflow_type: &str,
2667        task_queue: &str,
2668        workflow_id: &str,
2669        input: T,
2670    ) -> Result<WorkflowHandle> {
2671        self.start_workflow_with_options(
2672            workflow_type,
2673            task_queue,
2674            workflow_id,
2675            WorkflowStartOptions::default(),
2676            input,
2677        )
2678        .await
2679    }
2680
2681    /// Start a workflow with explicit server-enforced execution and run
2682    /// deadlines.
2683    pub async fn start_workflow_with_options<T: Serialize>(
2684        &self,
2685        workflow_type: &str,
2686        task_queue: &str,
2687        workflow_id: &str,
2688        options: WorkflowStartOptions,
2689        input: T,
2690    ) -> Result<WorkflowHandle> {
2691        options.validate()?;
2692        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2693        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2694        let body = json!({
2695            "workflow_id": workflow_id,
2696            "workflow_type": workflow_type,
2697            "task_queue": task_queue,
2698            "input": input_envelope,
2699            "execution_timeout_seconds": options.execution_timeout_seconds,
2700            "run_timeout_seconds": options.run_timeout_seconds
2701        });
2702
2703        let data: Value = self
2704            .request_json(
2705                reqwest::Method::POST,
2706                "/workflows",
2707                RequestProtocol::ControlPlane,
2708                Some(&body),
2709            )
2710            .await?;
2711
2712        Ok(WorkflowHandle {
2713            client: self.clone(),
2714            workflow_id: data
2715                .get("workflow_id")
2716                .and_then(Value::as_str)
2717                .unwrap_or(workflow_id)
2718                .to_string(),
2719            run_id: data
2720                .get("run_id")
2721                .and_then(Value::as_str)
2722                .map(str::to_string),
2723            workflow_type: data
2724                .get("workflow_type")
2725                .and_then(Value::as_str)
2726                .unwrap_or(workflow_type)
2727                .to_string(),
2728        })
2729    }
2730
2731    pub async fn signal_workflow<T: Serialize>(
2732        &self,
2733        workflow_id: &str,
2734        signal_name: &str,
2735        input: T,
2736    ) -> Result<Value> {
2737        self.signal_workflow_target(workflow_id, None, signal_name, input)
2738            .await
2739    }
2740
2741    /// Append one idempotently identified item to an instance-scoped input stream.
2742    pub async fn append_message_stream<T: Serialize>(
2743        &self,
2744        workflow_id: &str,
2745        stream_name: &str,
2746        message_id: &str,
2747        input: T,
2748    ) -> Result<Value> {
2749        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2750        let body = json!({
2751            "message_id": message_id,
2752            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2753        });
2754        self.request_json(
2755            reqwest::Method::POST,
2756            &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2757            RequestProtocol::ControlPlane,
2758            Some(&body),
2759        )
2760        .await
2761    }
2762
2763    /// Signal only if `run_id` is still the current run for this instance.
2764    pub async fn signal_workflow_run<T: Serialize>(
2765        &self,
2766        workflow_id: &str,
2767        run_id: &str,
2768        signal_name: &str,
2769        input: T,
2770    ) -> Result<Value> {
2771        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2772            .await
2773    }
2774
2775    async fn signal_workflow_target<T: Serialize>(
2776        &self,
2777        workflow_id: &str,
2778        run_id: Option<&str>,
2779        signal_name: &str,
2780        input: T,
2781    ) -> Result<Value> {
2782        validate_user_signal_name(signal_name)?;
2783        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2784        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2785        let body = json!({
2786            "input": input_envelope
2787        });
2788        let path = match run_id {
2789            Some(run_id) => {
2790                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2791            }
2792            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2793        };
2794        self.request_json(
2795            reqwest::Method::POST,
2796            &path,
2797            RequestProtocol::ControlPlane,
2798            Some(&body),
2799        )
2800        .await
2801    }
2802
2803    /// Request cooperative cancellation of the current run for an instance.
2804    pub async fn cancel_workflow(
2805        &self,
2806        workflow_id: &str,
2807        options: WorkflowCommandOptions,
2808    ) -> Result<WorkflowCommandResult> {
2809        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2810            .await
2811    }
2812
2813    /// Request cooperative cancellation only if `run_id` is still current.
2814    pub async fn cancel_workflow_run(
2815        &self,
2816        workflow_id: &str,
2817        run_id: &str,
2818        options: WorkflowCommandOptions,
2819    ) -> Result<WorkflowCommandResult> {
2820        self.workflow_command(
2821            workflow_id,
2822            Some(run_id),
2823            WorkflowCommandKind::Cancel,
2824            options,
2825        )
2826        .await
2827    }
2828
2829    /// Forcefully terminate the current run for an instance.
2830    pub async fn terminate_workflow(
2831        &self,
2832        workflow_id: &str,
2833        options: WorkflowCommandOptions,
2834    ) -> Result<WorkflowCommandResult> {
2835        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2836            .await
2837    }
2838
2839    /// Forcefully terminate only if `run_id` is still current.
2840    pub async fn terminate_workflow_run(
2841        &self,
2842        workflow_id: &str,
2843        run_id: &str,
2844        options: WorkflowCommandOptions,
2845    ) -> Result<WorkflowCommandResult> {
2846        self.workflow_command(
2847            workflow_id,
2848            Some(run_id),
2849            WorkflowCommandKind::Terminate,
2850            options,
2851        )
2852        .await
2853    }
2854
2855    async fn workflow_command(
2856        &self,
2857        workflow_id: &str,
2858        run_id: Option<&str>,
2859        command: WorkflowCommandKind,
2860        options: WorkflowCommandOptions,
2861    ) -> Result<WorkflowCommandResult> {
2862        let path = match run_id {
2863            Some(run_id) => format!(
2864                "/workflows/{workflow_id}/runs/{run_id}/{}",
2865                command.as_str()
2866            ),
2867            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2868        };
2869        let data = match self
2870            .request_json(
2871                reqwest::Method::POST,
2872                &path,
2873                RequestProtocol::ControlPlane,
2874                Some(&options),
2875            )
2876            .await
2877        {
2878            Ok(data) => data,
2879            Err(Error::Http { status, body }) => {
2880                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2881                    command,
2882                    status,
2883                    body,
2884                    workflow_id,
2885                    run_id,
2886                )));
2887            }
2888            Err(error) => return Err(error),
2889        };
2890
2891        Ok(workflow_command_result(command, data, workflow_id, run_id))
2892    }
2893
2894    /// Execute a named, read-only query against a running or completed workflow.
2895    ///
2896    /// Arguments and results use the platform payload envelope. Server and
2897    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
2898    /// reason, HTTP status, and original response body.
2899    pub async fn query_workflow<T: Serialize>(
2900        &self,
2901        workflow_id: &str,
2902        query_name: &str,
2903        input: T,
2904    ) -> Result<Value> {
2905        self.query_workflow_target(workflow_id, None, query_name, input)
2906            .await
2907    }
2908
2909    /// Query only if `run_id` is still current, preventing accidental retargeting.
2910    pub async fn query_workflow_run<T: Serialize>(
2911        &self,
2912        workflow_id: &str,
2913        run_id: &str,
2914        query_name: &str,
2915        input: T,
2916    ) -> Result<Value> {
2917        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2918            .await
2919    }
2920
2921    /// Query a workflow and return the lossless fixed Avro Value result.
2922    pub async fn query_workflow_avro_value<T: Serialize>(
2923        &self,
2924        workflow_id: &str,
2925        query_name: &str,
2926        input: T,
2927    ) -> Result<AvroValue> {
2928        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2929            .await
2930    }
2931
2932    /// Query a selected run and return the lossless fixed Avro Value result.
2933    pub async fn query_workflow_run_avro_value<T: Serialize>(
2934        &self,
2935        workflow_id: &str,
2936        run_id: &str,
2937        query_name: &str,
2938        input: T,
2939    ) -> Result<AvroValue> {
2940        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2941            .await
2942    }
2943
2944    async fn query_workflow_avro_value_target<T: Serialize>(
2945        &self,
2946        workflow_id: &str,
2947        run_id: Option<&str>,
2948        query_name: &str,
2949        input: T,
2950    ) -> Result<AvroValue> {
2951        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2952        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2953        let path = match run_id {
2954            Some(run_id) => {
2955                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2956            }
2957            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2958        };
2959        let response: Value = match self
2960            .request_json(
2961                reqwest::Method::POST,
2962                &path,
2963                RequestProtocol::ControlPlane,
2964                Some(&body),
2965            )
2966            .await
2967        {
2968            Ok(response) => response,
2969            Err(Error::Http { status, body }) => {
2970                return Err(Error::QueryFailed(query_failure(status, body)));
2971            }
2972            Err(error) => return Err(error),
2973        };
2974
2975        let envelope = response
2976            .get("result_envelope")
2977            .filter(|envelope| !envelope.is_null())
2978            .ok_or_else(|| {
2979                Error::Codec(
2980                    "missing_payload_envelope: typed query result requires result_envelope"
2981                        .to_string(),
2982                )
2983            })?;
2984        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2985    }
2986
2987    async fn query_workflow_target<T: Serialize>(
2988        &self,
2989        workflow_id: &str,
2990        run_id: Option<&str>,
2991        query_name: &str,
2992        input: T,
2993    ) -> Result<Value> {
2994        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2995        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2996        let body = json!({
2997            "input": input_envelope
2998        });
2999        let path = match run_id {
3000            Some(run_id) => {
3001                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3002            }
3003            None => format!("/workflows/{workflow_id}/query/{query_name}"),
3004        };
3005        let response: Value = match self
3006            .request_json(
3007                reqwest::Method::POST,
3008                &path,
3009                RequestProtocol::ControlPlane,
3010                Some(&body),
3011            )
3012            .await
3013        {
3014            Ok(response) => response,
3015            Err(Error::Http { status, body }) => {
3016                return Err(Error::QueryFailed(query_failure(status, body)));
3017            }
3018            Err(error) => return Err(error),
3019        };
3020
3021        if let Some(envelope) = response
3022            .get("result_envelope")
3023            .filter(|envelope| !envelope.is_null())
3024        {
3025            return decode_wire_value(envelope, DEFAULT_CODEC);
3026        }
3027
3028        Ok(response.get("result").cloned().unwrap_or(Value::Null))
3029    }
3030
3031    /// Send a synchronous update using fixed Avro Value arguments.
3032    pub async fn update_workflow<T: Serialize>(
3033        &self,
3034        workflow_id: &str,
3035        update_name: &str,
3036        input: T,
3037        request_id: Option<&str>,
3038    ) -> Result<Value> {
3039        let response = self
3040            .update_workflow_response(workflow_id, update_name, input, request_id)
3041            .await?;
3042        if let Some(envelope) = response
3043            .get("result_envelope")
3044            .filter(|envelope| !envelope.is_null())
3045        {
3046            return decode_wire_value(envelope, DEFAULT_CODEC);
3047        }
3048        Ok(response.get("result").cloned().unwrap_or(response))
3049    }
3050
3051    /// Send a synchronous update and retain a bytes-capable Avro result.
3052    pub async fn update_workflow_avro_value<T: Serialize>(
3053        &self,
3054        workflow_id: &str,
3055        update_name: &str,
3056        input: T,
3057        request_id: Option<&str>,
3058    ) -> Result<AvroValue> {
3059        let response = self
3060            .update_workflow_response(workflow_id, update_name, input, request_id)
3061            .await?;
3062        let envelope = response
3063            .get("result_envelope")
3064            .filter(|envelope| !envelope.is_null())
3065            .ok_or_else(|| {
3066                Error::Codec(
3067                    "missing_payload_envelope: typed update result requires result_envelope"
3068                        .to_string(),
3069                )
3070            })?;
3071        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3072    }
3073
3074    async fn update_workflow_response<T: Serialize>(
3075        &self,
3076        workflow_id: &str,
3077        update_name: &str,
3078        input: T,
3079        request_id: Option<&str>,
3080    ) -> Result<Value> {
3081        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3082        let mut body = json!({
3083            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3084            "wait_for": "completed",
3085        });
3086        if let Some(request_id) = request_id {
3087            body["request_id"] = json!(request_id);
3088        }
3089        self.request_json(
3090            reqwest::Method::POST,
3091            &format!("/workflows/{workflow_id}/update/{update_name}"),
3092            RequestProtocol::ControlPlane,
3093            Some(&body),
3094        )
3095        .await
3096    }
3097
3098    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3099        let path = format!("/workflows/{workflow_id}");
3100        let mut data: WorkflowDescription = self
3101            .request_json(
3102                reqwest::Method::GET,
3103                &path,
3104                RequestProtocol::ControlPlane,
3105                Option::<&Value>::None,
3106            )
3107            .await?;
3108        data.decode_payloads()?;
3109        Ok(data)
3110    }
3111
3112    /// Describe one selected run, including historical terminal runs.
3113    pub async fn describe_workflow_run(
3114        &self,
3115        workflow_id: &str,
3116        run_id: &str,
3117    ) -> Result<WorkflowDescription> {
3118        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3119        let mut data: WorkflowDescription = self
3120            .request_json(
3121                reqwest::Method::GET,
3122                &path,
3123                RequestProtocol::ControlPlane,
3124                Option::<&Value>::None,
3125            )
3126            .await?;
3127        data.decode_payloads()?;
3128        Ok(data)
3129    }
3130
3131    fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3132        let mut path = format!(
3133            "/workflows/{}/runs/{}/streams",
3134            percent_encode_path_segment(workflow_id),
3135            percent_encode_path_segment(run_id),
3136        );
3137        if let Some(stream_name) = stream_name {
3138            path.push('/');
3139            path.push_str(&percent_encode_path_segment(stream_name));
3140        }
3141        path
3142    }
3143
3144    /// List the run-scoped output streams already opened by a workflow.
3145    pub async fn list_workflow_streams(
3146        &self,
3147        workflow_id: &str,
3148        run_id: &str,
3149    ) -> Result<Vec<WorkflowStreamDescription>> {
3150        let response: WorkflowStreamListResponse = self
3151            .request_json(
3152                reqwest::Method::GET,
3153                &Self::workflow_stream_path(workflow_id, run_id, None),
3154                RequestProtocol::ControlPlane,
3155                Option::<&Value>::None,
3156            )
3157            .await?;
3158        Ok(response.streams)
3159    }
3160
3161    /// Describe stream lifecycle, offsets, pending count, and terminal error.
3162    pub async fn describe_workflow_stream(
3163        &self,
3164        workflow_id: &str,
3165        run_id: &str,
3166        stream_name: &str,
3167    ) -> Result<WorkflowStreamDescription> {
3168        let response: WorkflowStreamDescriptionResponse = self
3169            .request_json(
3170                reqwest::Method::GET,
3171                &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3172                RequestProtocol::ControlPlane,
3173                Option::<&Value>::None,
3174            )
3175            .await?;
3176        Ok(response.stream)
3177    }
3178
3179    /// Read one bounded page beginning at a zero-based offset.
3180    ///
3181    /// Delivery is at least once: persist `next_offset` only after processing
3182    /// the page. The future is cancellation-safe; dropping it cancels the
3183    /// in-flight request. Long polling is capped at 60 seconds by the SDK and
3184    /// service contract.
3185    pub async fn subscribe_workflow_stream(
3186        &self,
3187        workflow_id: &str,
3188        run_id: &str,
3189        stream_name: &str,
3190        from_offset: u64,
3191        max_items: usize,
3192        wait: Duration,
3193    ) -> Result<WorkflowStreamPage> {
3194        let max_items = max_items.clamp(1, 500);
3195        let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3196        let path = format!(
3197            "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3198            Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3199        );
3200        let response: WorkflowStreamPageResponse = self
3201            .request_json_with_timeout(
3202                reqwest::Method::GET,
3203                &path,
3204                RequestProtocol::ControlPlane,
3205                Option::<&Value>::None,
3206                Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3207            )
3208            .await?;
3209
3210        let items = response
3211            .items
3212            .into_iter()
3213            .map(|raw| {
3214                let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3215                let envelope = raw.get("payload").cloned();
3216                let payload = envelope
3217                    .as_ref()
3218                    .filter(|value| value.get("blob").is_some())
3219                    .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3220                    .transpose()?
3221                    .map(AvroValue::into_json)
3222                    .transpose()?;
3223                Ok(WorkflowStreamItem {
3224                    offset,
3225                    payload,
3226                    payload_envelope: envelope,
3227                    payload_reference: raw
3228                        .get("payload_reference")
3229                        .and_then(Value::as_str)
3230                        .map(str::to_string),
3231                    payload_codec: raw
3232                        .get("payload_codec")
3233                        .and_then(Value::as_str)
3234                        .map(str::to_string),
3235                    idempotency_key: raw
3236                        .get("idempotency_key")
3237                        .and_then(Value::as_str)
3238                        .map(str::to_string),
3239                    item_type: raw
3240                        .get("item_type")
3241                        .and_then(Value::as_str)
3242                        .map(str::to_string),
3243                    content_type: raw
3244                        .get("content_type")
3245                        .and_then(Value::as_str)
3246                        .map(str::to_string),
3247                    origin: raw
3248                        .get("origin")
3249                        .and_then(Value::as_str)
3250                        .map(str::to_string),
3251                    origin_reference: raw
3252                        .get("origin_reference")
3253                        .and_then(Value::as_str)
3254                        .map(str::to_string),
3255                    emitted_at: raw
3256                        .get("emitted_at")
3257                        .and_then(Value::as_str)
3258                        .map(str::to_string),
3259                    raw,
3260                })
3261            })
3262            .collect::<Result<Vec<_>>>()?;
3263        Ok(WorkflowStreamPage {
3264            stream: response.stream,
3265            items,
3266            next_offset: response.next_offset,
3267            terminal: response.terminal,
3268        })
3269    }
3270
3271    /// Append inline Avro envelopes or opaque external payload references.
3272    pub async fn append_workflow_stream(
3273        &self,
3274        workflow_id: &str,
3275        run_id: &str,
3276        stream_name: &str,
3277        items: &[WorkflowStreamAppendItem],
3278        max_pending_items: Option<u64>,
3279    ) -> Result<WorkflowStreamAppendResult> {
3280        if items.is_empty() {
3281            return Err(Error::Codec(
3282                "workflow_stream_items_empty: append requires at least one item".to_string(),
3283            ));
3284        }
3285        let mut body = json!({
3286            "items": items
3287                .iter()
3288                .map(|item| item.wire_value(None))
3289                .collect::<Vec<_>>(),
3290        });
3291        if let Some(max_pending_items) = max_pending_items {
3292            if max_pending_items == 0 {
3293                return Err(Error::Codec(
3294                    "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3295                        .to_string(),
3296                ));
3297            }
3298            body["max_pending_items"] = json!(max_pending_items);
3299        }
3300        let response: WorkflowStreamAppendResponse = self
3301            .request_json(
3302                reqwest::Method::POST,
3303                &format!(
3304                    "{}/items",
3305                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3306                ),
3307                RequestProtocol::ControlPlane,
3308                Some(&body),
3309            )
3310            .await?;
3311        Ok(WorkflowStreamAppendResult {
3312            stream: response.stream,
3313            accepted_offsets: response.accepted_offsets,
3314            accepted: response.accepted,
3315            deduped: response.deduped,
3316        })
3317    }
3318
3319    /// Close a stream, or mark it errored when `error_reason` is supplied.
3320    pub async fn close_workflow_stream(
3321        &self,
3322        workflow_id: &str,
3323        run_id: &str,
3324        stream_name: &str,
3325        error_reason: Option<&str>,
3326        retention_seconds: Option<u64>,
3327    ) -> Result<WorkflowStreamDescription> {
3328        let mut body = json!({});
3329        if let Some(error_reason) = error_reason {
3330            body["error_reason"] = json!(error_reason);
3331        }
3332        if let Some(retention_seconds) = retention_seconds {
3333            if retention_seconds == 0 {
3334                return Err(Error::Codec(
3335                    "workflow_stream_retention_invalid: retention_seconds must be positive"
3336                        .to_string(),
3337                ));
3338            }
3339            body["retention_seconds"] = json!(retention_seconds);
3340        }
3341        let response: WorkflowStreamDescriptionResponse = self
3342            .request_json(
3343                reqwest::Method::POST,
3344                &format!(
3345                    "{}/close",
3346                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3347                ),
3348                RequestProtocol::ControlPlane,
3349                Some(&body),
3350            )
3351            .await?;
3352        Ok(response.stream)
3353    }
3354
3355    pub async fn register_worker(
3356        &self,
3357        worker_id: &str,
3358        task_queue: &str,
3359        supported_workflow_types: Vec<String>,
3360        supported_activity_types: Vec<String>,
3361        max_concurrent_workflow_tasks: usize,
3362        max_concurrent_activity_tasks: usize,
3363    ) -> Result<RegisterWorkerResponse> {
3364        self.register_worker_with_capabilities(
3365            worker_id,
3366            task_queue,
3367            supported_workflow_types,
3368            supported_activity_types,
3369            max_concurrent_workflow_tasks,
3370            max_concurrent_activity_tasks,
3371            Vec::new(),
3372        )
3373        .await
3374    }
3375
3376    /// Register a worker and explicitly advertise additive worker capabilities.
3377    pub async fn register_worker_with_capabilities(
3378        &self,
3379        worker_id: &str,
3380        task_queue: &str,
3381        supported_workflow_types: Vec<String>,
3382        supported_activity_types: Vec<String>,
3383        max_concurrent_workflow_tasks: usize,
3384        max_concurrent_activity_tasks: usize,
3385        capabilities: Vec<String>,
3386    ) -> Result<RegisterWorkerResponse> {
3387        self.register_worker_with_command_contracts(
3388            worker_id,
3389            task_queue,
3390            supported_workflow_types,
3391            supported_activity_types,
3392            max_concurrent_workflow_tasks,
3393            max_concurrent_activity_tasks,
3394            capabilities,
3395            Value::Object(serde_json::Map::new()),
3396        )
3397        .await
3398    }
3399
3400    /// Register a worker and advertise its named query and update handlers.
3401    ///
3402    /// This Rust SDK cannot execute synchronous pre-accept update validation,
3403    /// so a workflow contract with a non-empty or malformed
3404    /// `update_validators` declaration returns
3405    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
3406    #[allow(clippy::too_many_arguments)]
3407    pub async fn register_worker_with_command_contracts(
3408        &self,
3409        worker_id: &str,
3410        task_queue: &str,
3411        supported_workflow_types: Vec<String>,
3412        supported_activity_types: Vec<String>,
3413        max_concurrent_workflow_tasks: usize,
3414        max_concurrent_activity_tasks: usize,
3415        capabilities: Vec<String>,
3416        workflow_command_contracts: Value,
3417    ) -> Result<RegisterWorkerResponse> {
3418        if let Some(contracts) = workflow_command_contracts.as_object() {
3419            for (workflow_type, contract) in contracts {
3420                let Some(update_validators) = contract.get("update_validators") else {
3421                    continue;
3422                };
3423                if !update_validators
3424                    .as_array()
3425                    .is_some_and(|validators| validators.is_empty())
3426                {
3427                    return Err(Error::UnsupportedUpdateValidators {
3428                        workflow_type: workflow_type.clone(),
3429                    });
3430                }
3431            }
3432        }
3433
3434        let mut body = json!({
3435            "worker_id": worker_id,
3436            "task_queue": task_queue,
3437            "runtime": "rust",
3438            "sdk_version": SDK_VERSION,
3439            "supported_workflow_types": supported_workflow_types,
3440            "supported_activity_types": supported_activity_types,
3441            "capabilities": capabilities,
3442            "capability_manifest": portable_worker_affinity_capability_manifest(),
3443            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3444            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3445        });
3446        if workflow_command_contracts
3447            .as_object()
3448            .is_some_and(|contracts| !contracts.is_empty())
3449        {
3450            body["workflow_command_contracts"] = workflow_command_contracts;
3451        }
3452
3453        self.request_json(
3454            reqwest::Method::POST,
3455            "/worker/register",
3456            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3457            Some(&body),
3458        )
3459        .await
3460    }
3461
3462    /// Gracefully remove one worker's registration through the worker plane.
3463    ///
3464    /// This operation is separate from operator-facing worker management. It
3465    /// uses worker-protocol authentication and returns the server's lease
3466    /// recovery result.
3467    pub async fn deregister_worker_registration(
3468        &self,
3469        worker_id: &str,
3470    ) -> Result<WorkerDeregistrationEnvelope> {
3471        let path = format!(
3472            "/worker/registrations/{}",
3473            percent_encode_path_segment(worker_id)
3474        );
3475        self.request_json(
3476            reqwest::Method::DELETE,
3477            &path,
3478            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3479            Option::<&Value>::None,
3480        )
3481        .await
3482    }
3483
3484    /// Long-poll for an ephemeral, read-only workflow query task.
3485    pub async fn poll_query_task(
3486        &self,
3487        worker_id: &str,
3488        task_queue: &str,
3489        timeout: Duration,
3490    ) -> Result<Option<QueryTask>> {
3491        Ok(self
3492            .poll_query_task_response(worker_id, task_queue, timeout)
3493            .await?
3494            .task)
3495    }
3496
3497    /// Poll a query task while preserving server stop and drain metadata.
3498    pub async fn poll_query_task_response(
3499        &self,
3500        worker_id: &str,
3501        task_queue: &str,
3502        timeout: Duration,
3503    ) -> Result<PollQueryTaskResponse> {
3504        let poll_request_id = unique_request_id("rust-query-poll");
3505        self.poll_query_task_response_with_request_id(
3506            worker_id,
3507            task_queue,
3508            timeout,
3509            &poll_request_id,
3510            1,
3511        )
3512        .await
3513    }
3514
3515    async fn poll_query_task_response_with_request_id(
3516        &self,
3517        worker_id: &str,
3518        task_queue: &str,
3519        timeout: Duration,
3520        poll_request_id: &str,
3521        transport_retries: usize,
3522    ) -> Result<PollQueryTaskResponse> {
3523        let timeout_seconds = long_poll_timeout_seconds(timeout);
3524        let body = json!({
3525            "worker_id": worker_id,
3526            "task_queue": task_queue,
3527            "poll_request_id": poll_request_id,
3528            "timeout_seconds": timeout_seconds,
3529        });
3530        self.poll_request_json(
3531            "/worker/query-tasks/poll",
3532            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3533            &body,
3534            timeout + Duration::from_secs(5),
3535            transport_retries,
3536        )
3537        .await
3538    }
3539
3540    /// Complete a query task without appending workflow history.
3541    pub async fn complete_query_task<T: Serialize>(
3542        &self,
3543        query_task_id: &str,
3544        lease_owner: &str,
3545        query_task_attempt: u64,
3546        result: T,
3547        codec: &str,
3548    ) -> Result<Value> {
3549        let typed_result = AvroValue::from_serialize(&result)?;
3550        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3551        self.complete_query_task_with_envelope(
3552            query_task_id,
3553            lease_owner,
3554            query_task_attempt,
3555            typed_result.into_json()?,
3556            result_envelope,
3557        )
3558        .await
3559    }
3560
3561    async fn complete_query_task_with_envelope(
3562        &self,
3563        query_task_id: &str,
3564        lease_owner: &str,
3565        query_task_attempt: u64,
3566        result: Value,
3567        result_envelope: Value,
3568    ) -> Result<Value> {
3569        let body = json!({
3570            "lease_owner": lease_owner,
3571            "query_task_attempt": query_task_attempt,
3572            "result": result,
3573            "result_envelope": result_envelope,
3574        });
3575        let path = format!("/worker/query-tasks/{query_task_id}/complete");
3576        let response = self
3577            .request_json(
3578                reqwest::Method::POST,
3579                &path,
3580                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3581                Some(&body),
3582            )
3583            .await;
3584        query_task_response(response)
3585    }
3586
3587    /// Report a stable machine-readable query-task failure.
3588    pub async fn fail_query_task(
3589        &self,
3590        query_task_id: &str,
3591        lease_owner: &str,
3592        query_task_attempt: u64,
3593        message: impl Into<String>,
3594        reason: impl Into<String>,
3595        failure_type: impl Into<String>,
3596    ) -> Result<Value> {
3597        let body = json!({
3598            "lease_owner": lease_owner,
3599            "query_task_attempt": query_task_attempt,
3600            "failure": {
3601                "message": message.into(),
3602                "reason": reason.into(),
3603                "type": failure_type.into(),
3604            }
3605        });
3606        let path = format!("/worker/query-tasks/{query_task_id}/fail");
3607        let response = self
3608            .request_json(
3609                reqwest::Method::POST,
3610                &path,
3611                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3612                Some(&body),
3613            )
3614            .await;
3615        query_task_response(response)
3616    }
3617
3618    pub async fn heartbeat_worker(
3619        &self,
3620        worker_id: &str,
3621        workflow_available: usize,
3622        activity_available: usize,
3623    ) -> Result<Value> {
3624        let body = json!({
3625            "worker_id": worker_id,
3626            "task_slots": {
3627                "workflow_available": workflow_available,
3628                "activity_available": activity_available
3629            },
3630            "process_metrics": {
3631                "process_id": std::process::id(),
3632                "process_uptime_seconds": 0
3633            }
3634        });
3635
3636        self.request_json(
3637            reqwest::Method::POST,
3638            "/worker/heartbeat",
3639            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3640            Some(&body),
3641        )
3642        .await
3643    }
3644
3645    pub async fn poll_workflow_task(
3646        &self,
3647        worker_id: &str,
3648        task_queue: &str,
3649        timeout: Duration,
3650    ) -> Result<Option<WorkflowTask>> {
3651        Ok(self
3652            .poll_workflow_task_response(worker_id, task_queue, timeout)
3653            .await?
3654            .task)
3655    }
3656
3657    pub async fn poll_workflow_task_response(
3658        &self,
3659        worker_id: &str,
3660        task_queue: &str,
3661        timeout: Duration,
3662    ) -> Result<PollWorkflowTaskResponse> {
3663        let poll_request_id = unique_request_id("rust-workflow-poll");
3664        self.poll_workflow_task_response_with_request_id(
3665            worker_id,
3666            task_queue,
3667            timeout,
3668            &poll_request_id,
3669            1,
3670        )
3671        .await
3672    }
3673
3674    async fn poll_workflow_task_response_with_request_id(
3675        &self,
3676        worker_id: &str,
3677        task_queue: &str,
3678        timeout: Duration,
3679        poll_request_id: &str,
3680        transport_retries: usize,
3681    ) -> Result<PollWorkflowTaskResponse> {
3682        let body = json!({
3683            "worker_id": worker_id,
3684            "task_queue": task_queue,
3685            "poll_request_id": poll_request_id,
3686            "timeout_seconds": long_poll_timeout_seconds(timeout),
3687        });
3688        let mut data: PollWorkflowTaskResponse = self
3689            .poll_request_json(
3690                "/worker/workflow-tasks/poll",
3691                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3692                &body,
3693                timeout + Duration::from_secs(5),
3694                transport_retries,
3695            )
3696            .await?;
3697
3698        if let Some(task) = data.task.as_mut() {
3699            self.fetch_remaining_workflow_history(worker_id, task)
3700                .await?;
3701        }
3702
3703        Ok(data)
3704    }
3705
3706    async fn fetch_remaining_workflow_history(
3707        &self,
3708        worker_id: &str,
3709        task: &mut WorkflowTask,
3710    ) -> Result<()> {
3711        let mut next_token = task.next_history_page_token.clone();
3712
3713        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3714            let lease_owner = task
3715                .lease_owner
3716                .clone()
3717                .unwrap_or_else(|| worker_id.to_string());
3718            let page = self
3719                .workflow_task_history_page(
3720                    &task.task_id,
3721                    &lease_owner,
3722                    task.workflow_task_attempt,
3723                    &token,
3724                )
3725                .await?;
3726
3727            task.append_history_page(page);
3728
3729            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3730                return Err(Error::Codec(
3731                    "workflow history pagination returned the same page token".to_string(),
3732                ));
3733            }
3734
3735            next_token = task.next_history_page_token.clone();
3736        }
3737
3738        Ok(())
3739    }
3740
3741    async fn workflow_task_history_page(
3742        &self,
3743        task_id: &str,
3744        lease_owner: &str,
3745        workflow_task_attempt: u64,
3746        next_history_page_token: &str,
3747    ) -> Result<WorkflowTaskHistoryPage> {
3748        let body = json!({
3749            "lease_owner": lease_owner,
3750            "workflow_task_attempt": workflow_task_attempt,
3751            "next_history_page_token": next_history_page_token
3752        });
3753        let path = format!("/worker/workflow-tasks/{task_id}/history");
3754
3755        self.request_json(
3756            reqwest::Method::POST,
3757            &path,
3758            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3759            Some(&body),
3760        )
3761        .await
3762    }
3763
3764    pub async fn complete_workflow_task(
3765        &self,
3766        task_id: &str,
3767        lease_owner: &str,
3768        workflow_task_attempt: u64,
3769        commands: Vec<Value>,
3770    ) -> Result<Value> {
3771        self.complete_workflow_task_with_message_streams(
3772            task_id,
3773            lease_owner,
3774            workflow_task_attempt,
3775            commands,
3776            Vec::new(),
3777            Vec::new(),
3778        )
3779        .await
3780    }
3781
3782    async fn complete_workflow_task_with_message_streams(
3783        &self,
3784        task_id: &str,
3785        lease_owner: &str,
3786        workflow_task_attempt: u64,
3787        commands: Vec<Value>,
3788        message_stream_cursors: Vec<Value>,
3789        message_stream_waits: Vec<Value>,
3790    ) -> Result<Value> {
3791        validate_workflow_task_commands(&commands)?;
3792        let has_message_stream_metadata =
3793            !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3794        if has_message_stream_metadata
3795            && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3796        {
3797            return Err(Error::Codec(
3798                "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3799                    .to_string(),
3800            ));
3801        }
3802        let protocol_version = workflow_completion_protocol_version_with_message_streams(
3803            &commands,
3804            has_message_stream_metadata,
3805        );
3806        let mut body = json!({
3807            "lease_owner": lease_owner,
3808            "workflow_task_attempt": workflow_task_attempt,
3809            "commands": commands
3810        });
3811        if !message_stream_cursors.is_empty() {
3812            body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3813        }
3814        if !message_stream_waits.is_empty() {
3815            body["message_stream_waits"] = Value::Array(message_stream_waits);
3816        }
3817        let path = format!("/worker/workflow-tasks/{task_id}/complete");
3818        self.request_json(
3819            reqwest::Method::POST,
3820            &path,
3821            RequestProtocol::Worker(protocol_version),
3822            Some(&body),
3823        )
3824        .await
3825    }
3826
3827    pub async fn fail_workflow_task(
3828        &self,
3829        task_id: &str,
3830        lease_owner: &str,
3831        workflow_task_attempt: u64,
3832        message: impl Into<String>,
3833    ) -> Result<Value> {
3834        self.fail_workflow_task_with_type(
3835            task_id,
3836            lease_owner,
3837            workflow_task_attempt,
3838            message,
3839            "RustWorkflowTaskFailure",
3840        )
3841        .await
3842    }
3843
3844    async fn fail_workflow_task_with_type(
3845        &self,
3846        task_id: &str,
3847        lease_owner: &str,
3848        workflow_task_attempt: u64,
3849        message: impl Into<String>,
3850        failure_type: &str,
3851    ) -> Result<Value> {
3852        let body = json!({
3853            "lease_owner": lease_owner,
3854            "workflow_task_attempt": workflow_task_attempt,
3855            "failure": {
3856                "message": message.into(),
3857                "type": failure_type
3858            }
3859        });
3860        let path = format!("/worker/workflow-tasks/{task_id}/fail");
3861        self.request_json(
3862            reqwest::Method::POST,
3863            &path,
3864            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3865            Some(&body),
3866        )
3867        .await
3868    }
3869
3870    pub async fn poll_activity_task(
3871        &self,
3872        worker_id: &str,
3873        task_queue: &str,
3874        timeout: Duration,
3875    ) -> Result<Option<ActivityTask>> {
3876        Ok(self
3877            .poll_activity_task_response(worker_id, task_queue, timeout)
3878            .await?
3879            .task)
3880    }
3881
3882    /// Poll an activity task while preserving server stop and drain metadata.
3883    pub async fn poll_activity_task_response(
3884        &self,
3885        worker_id: &str,
3886        task_queue: &str,
3887        timeout: Duration,
3888    ) -> Result<PollActivityTaskResponse> {
3889        let poll_request_id = unique_request_id("rust-activity-poll");
3890        self.poll_activity_task_response_with_request_id(
3891            worker_id,
3892            task_queue,
3893            timeout,
3894            &poll_request_id,
3895            1,
3896        )
3897        .await
3898    }
3899
3900    async fn poll_activity_task_response_with_request_id(
3901        &self,
3902        worker_id: &str,
3903        task_queue: &str,
3904        timeout: Duration,
3905        poll_request_id: &str,
3906        transport_retries: usize,
3907    ) -> Result<PollActivityTaskResponse> {
3908        let body = json!({
3909            "worker_id": worker_id,
3910            "task_queue": task_queue,
3911            "poll_request_id": poll_request_id,
3912            "timeout_seconds": long_poll_timeout_seconds(timeout),
3913        });
3914        let data: PollActivityTaskResponse = self
3915            .poll_request_json(
3916                "/worker/activity-tasks/poll",
3917                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3918                &body,
3919                timeout + Duration::from_secs(5),
3920                transport_retries,
3921            )
3922            .await?;
3923        Ok(data)
3924    }
3925
3926    pub async fn complete_activity_task<T: Serialize>(
3927        &self,
3928        task_id: &str,
3929        activity_attempt_id: &str,
3930        lease_owner: &str,
3931        result: T,
3932        codec: &str,
3933    ) -> Result<Value> {
3934        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3935        let body = json!({
3936            "activity_attempt_id": activity_attempt_id,
3937            "lease_owner": lease_owner,
3938            "result": result
3939        });
3940        let path = format!("/worker/activity-tasks/{task_id}/complete");
3941        activity_task_response(
3942            self.request_json(
3943                reqwest::Method::POST,
3944                &path,
3945                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3946                Some(&body),
3947            )
3948            .await,
3949            "complete",
3950            task_id,
3951            activity_attempt_id,
3952        )
3953    }
3954
3955    pub async fn fail_activity_task(
3956        &self,
3957        task_id: &str,
3958        activity_attempt_id: &str,
3959        lease_owner: &str,
3960        message: impl Into<String>,
3961        non_retryable: bool,
3962    ) -> Result<Value> {
3963        let body = json!({
3964            "activity_attempt_id": activity_attempt_id,
3965            "lease_owner": lease_owner,
3966            "failure": {
3967                "message": message.into(),
3968                "type": "RustActivityFailure",
3969                "non_retryable": non_retryable
3970            }
3971        });
3972        let path = format!("/worker/activity-tasks/{task_id}/fail");
3973        activity_task_response(
3974            self.request_json(
3975                reqwest::Method::POST,
3976                &path,
3977                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3978                Some(&body),
3979            )
3980            .await,
3981            "fail",
3982            task_id,
3983            activity_attempt_id,
3984        )
3985    }
3986
3987    pub async fn heartbeat_activity_task<T: Serialize>(
3988        &self,
3989        task_id: &str,
3990        activity_attempt_id: &str,
3991        lease_owner: &str,
3992        details: T,
3993    ) -> Result<ActivityHeartbeatResponse> {
3994        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3995        let body = json!({
3996            "activity_attempt_id": activity_attempt_id,
3997            "lease_owner": lease_owner,
3998            "details": details
3999        });
4000        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4001        activity_task_response(
4002            self.request_json(
4003                reqwest::Method::POST,
4004                &path,
4005                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4006                Some(&body),
4007            )
4008            .await,
4009            "heartbeat",
4010            task_id,
4011            activity_attempt_id,
4012        )
4013    }
4014
4015    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4016        &self,
4017        method: reqwest::Method,
4018        path: &str,
4019        protocol: RequestProtocol,
4020        body: Option<&B>,
4021    ) -> Result<T> {
4022        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4023            .await
4024    }
4025
4026    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4027        &self,
4028        method: reqwest::Method,
4029        path: &str,
4030        protocol: RequestProtocol,
4031        body: Option<&B>,
4032        timeout: Duration,
4033    ) -> Result<T> {
4034        let auth_token = self.auth_token(protocol)?;
4035        let mut request = self
4036            .http
4037            .request(method, format!("{}/api{}", self.base_url, path))
4038            .timeout(timeout)
4039            .header(reqwest::header::ACCEPT, "application/json")
4040            .header(reqwest::header::CONTENT_TYPE, "application/json")
4041            .header("X-Namespace", &self.namespace);
4042
4043        match protocol {
4044            RequestProtocol::Worker(version) => {
4045                request = request.header("X-Durable-Workflow-Protocol-Version", version);
4046            }
4047            RequestProtocol::ControlPlane => {
4048                request = request.header(
4049                    "X-Durable-Workflow-Control-Plane-Version",
4050                    CONTROL_PLANE_VERSION,
4051                );
4052            }
4053        }
4054
4055        if let Some(token) = auth_token {
4056            request = request.bearer_auth(token);
4057        }
4058
4059        if let Some(body) = body {
4060            request = request.json(body);
4061        }
4062
4063        let request = request.build()?;
4064        let admission = self
4065            .worker_storage_admission
4066            .as_ref()
4067            .filter(|_| matches!(protocol, RequestProtocol::Worker(_)));
4068        let poll_request_id = path.ends_with("/poll").then(|| {
4069            request
4070                .body()
4071                .and_then(reqwest::Body::as_bytes)
4072                .and_then(|body| serde_json::from_slice::<Value>(body).ok())
4073                .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4074                .unwrap_or_default()
4075        });
4076        let mut storage_retries = 0_usize;
4077
4078        loop {
4079            // Keep the serialized body and lease/poll identity across admission pauses.
4080            let response = self
4081                .http
4082                .execute(request.try_clone().ok_or_else(|| {
4083                    Error::WorkerLoop("worker request body cannot be retried".to_string())
4084                })?)
4085                .await?;
4086            let status = response.status();
4087            let bytes = response.bytes().await?;
4088
4089            if !status.is_success() {
4090                let body = String::from_utf8_lossy(&bytes).to_string();
4091                if let Some(protocol) = protocol_failure(status, &body) {
4092                    return Err(Error::Protocol(protocol));
4093                }
4094                let error = Error::Http { status, body };
4095                if let Some(admission) = admission {
4096                    if let Some(advertised_delay) =
4097                        worker_storage_admission_retry_after(&error, poll_request_id.as_deref())
4098                    {
4099                        storage_retries = storage_retries.saturating_add(1);
4100                        let delay = worker_retry_delay(admission.policy, storage_retries)
4101                            .max(advertised_delay)
4102                            .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4103                        let deadline = tokio::time::Instant::now() + delay;
4104                        loop {
4105                            if admission.stop.load(Ordering::SeqCst) {
4106                                return Err(error);
4107                            }
4108                            let remaining =
4109                                deadline.saturating_duration_since(tokio::time::Instant::now());
4110                            if remaining.is_zero() {
4111                                break;
4112                            }
4113                            tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4114                        }
4115                        continue;
4116                    }
4117                }
4118                return Err(error);
4119            }
4120
4121            if bytes.is_empty() {
4122                return Ok(serde_json::from_value(Value::Null)?);
4123            }
4124
4125            let mut value: Value = serde_json::from_slice(&bytes)?;
4126            self.resolve_runtime_payloads(&mut value, path, protocol)
4127                .await?;
4128            return Ok(serde_json::from_value(value)?);
4129        }
4130    }
4131
4132    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4133        &self,
4134        path: &str,
4135        protocol: RequestProtocol,
4136        body: &B,
4137        timeout: Duration,
4138        max_retries: usize,
4139    ) -> Result<T> {
4140        let mut retries = 0;
4141
4142        loop {
4143            let response = self
4144                .request_json_with_timeout(
4145                    reqwest::Method::POST,
4146                    path,
4147                    protocol,
4148                    Some(body),
4149                    timeout,
4150                )
4151                .await;
4152
4153            match response {
4154                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4155                response => return worker_poll_response(response),
4156            }
4157        }
4158    }
4159
4160    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4161        match protocol {
4162            RequestProtocol::Worker(_) => {
4163                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4164                    return Ok(Some(token));
4165                }
4166                if self.control_token.is_some() {
4167                    return Err(Error::MissingRoleCredentials {
4168                        role: "worker",
4169                        opposite_role: "control",
4170                    });
4171                }
4172                Ok(None)
4173            }
4174            RequestProtocol::ControlPlane => {
4175                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4176                    return Ok(Some(token));
4177                }
4178                if self.worker_token.is_some() {
4179                    return Err(Error::MissingRoleCredentials {
4180                        role: "control",
4181                        opposite_role: "worker",
4182                    });
4183                }
4184                Ok(None)
4185            }
4186        }
4187    }
4188}
4189
4190fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4191    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4192    let reason = body
4193        .get("reason")
4194        .and_then(Value::as_str)
4195        .unwrap_or("query_rejected")
4196        .to_string();
4197    let message = body
4198        .get("message")
4199        .or_else(|| body.get("error"))
4200        .and_then(Value::as_str)
4201        .unwrap_or("workflow query was rejected")
4202        .to_string();
4203
4204    QueryFailure {
4205        status: status.as_u16(),
4206        reason,
4207        message,
4208        body,
4209    }
4210}
4211
4212fn workflow_command_result(
4213    command: WorkflowCommandKind,
4214    data: Value,
4215    workflow_id: &str,
4216    run_id: Option<&str>,
4217) -> WorkflowCommandResult {
4218    WorkflowCommandResult {
4219        command,
4220        workflow_id: data
4221            .get("workflow_id")
4222            .and_then(Value::as_str)
4223            .unwrap_or(workflow_id)
4224            .to_string(),
4225        run_id: data
4226            .get("run_id")
4227            .and_then(Value::as_str)
4228            .or(run_id)
4229            .map(str::to_string),
4230        outcome: data
4231            .get("outcome")
4232            .and_then(Value::as_str)
4233            .map(str::to_string),
4234        reason: data
4235            .get("reason")
4236            .and_then(Value::as_str)
4237            .map(str::to_string),
4238        command_status: data
4239            .get("command_status")
4240            .and_then(Value::as_str)
4241            .map(str::to_string),
4242        raw: data,
4243    }
4244}
4245
4246fn workflow_command_rejection(
4247    command: WorkflowCommandKind,
4248    status: reqwest::StatusCode,
4249    raw_body: String,
4250    workflow_id: &str,
4251    run_id: Option<&str>,
4252) -> WorkflowCommandRejection {
4253    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4254    WorkflowCommandRejection {
4255        command,
4256        status: status.as_u16(),
4257        reason: body
4258            .get("reason")
4259            .and_then(Value::as_str)
4260            .unwrap_or("workflow_command_rejected")
4261            .to_string(),
4262        message: body
4263            .get("message")
4264            .or_else(|| body.get("error"))
4265            .and_then(Value::as_str)
4266            .unwrap_or("workflow lifecycle command was rejected")
4267            .to_string(),
4268        workflow_id: body
4269            .get("workflow_id")
4270            .and_then(Value::as_str)
4271            .unwrap_or(workflow_id)
4272            .to_string(),
4273        run_id: body
4274            .get("run_id")
4275            .and_then(Value::as_str)
4276            .or(run_id)
4277            .map(str::to_string),
4278        target_scope: body
4279            .get("target_scope")
4280            .and_then(Value::as_str)
4281            .map(str::to_string),
4282        body,
4283    }
4284}
4285
4286fn query_task_response(response: Result<Value>) -> Result<Value> {
4287    match response {
4288        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4289        response => response,
4290    }
4291}
4292
4293fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4294    match response {
4295        Err(Error::Http { status, body })
4296            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4297        {
4298            Ok(serde_json::from_str(&body)?)
4299        }
4300        response => response,
4301    }
4302}
4303
4304fn worker_poll_body_is_stop(body: &str) -> bool {
4305    serde_json::from_str::<Value>(body)
4306        .ok()
4307        .is_some_and(|body| {
4308            worker_poll_is_stop(
4309                body.get("poll_status").and_then(Value::as_str),
4310                body.get("reason").and_then(Value::as_str),
4311            )
4312        })
4313}
4314
4315fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4316    matches!(poll_status, Some("draining" | "stopped"))
4317        || matches!(reason, Some("worker_draining" | "worker_stopped"))
4318}
4319
4320fn query_task_rejection_is_final(error: &Error) -> bool {
4321    matches!(
4322        error,
4323        Error::QueryFailed(failure)
4324            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4325    )
4326}
4327
4328fn activity_task_response<T>(
4329    response: Result<T>,
4330    operation: &str,
4331    task_id: &str,
4332    activity_attempt_id: &str,
4333) -> Result<T> {
4334    match response {
4335        Err(Error::Http { status, body }) => {
4336            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4337            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4338                operation: operation.to_string(),
4339                status: status.as_u16(),
4340                reason: body
4341                    .get("reason")
4342                    .and_then(Value::as_str)
4343                    .unwrap_or("activity_task_rejected")
4344                    .to_string(),
4345                task_id: body
4346                    .get("task_id")
4347                    .and_then(Value::as_str)
4348                    .unwrap_or(task_id)
4349                    .to_string(),
4350                activity_attempt_id: body
4351                    .get("activity_attempt_id")
4352                    .and_then(Value::as_str)
4353                    .unwrap_or(activity_attempt_id)
4354                    .to_string(),
4355                cancel_requested: body
4356                    .get("cancel_requested")
4357                    .and_then(Value::as_bool)
4358                    .unwrap_or(false),
4359                can_continue: body.get("can_continue").and_then(Value::as_bool),
4360                run_closed_reason: body
4361                    .get("run_closed_reason")
4362                    .and_then(Value::as_str)
4363                    .map(str::to_string),
4364                body,
4365            }))
4366        }
4367        response => response,
4368    }
4369}
4370
4371fn activity_task_rejection_is_final(error: &Error) -> bool {
4372    matches!(
4373        error,
4374        Error::ActivityTaskRejected(rejection)
4375            if matches!(
4376                rejection.reason.as_str(),
4377                "run_cancelled"
4378                    | "run_terminated"
4379                    | "attempt_closed"
4380                    | "stale_attempt"
4381                    | "activity_cancelled"
4382                    | "task_cancelled"
4383                    | "run_closed"
4384                    | "activity_not_running"
4385                    | "attempt_not_found"
4386            )
4387    )
4388}
4389
4390fn workflow_task_completion_is_terminal_timeout(
4391    error: &Error,
4392    task_id: &str,
4393    workflow_task_attempt: u64,
4394    run_id: Option<&str>,
4395) -> bool {
4396    let Error::Http { status, body } = error else {
4397        return false;
4398    };
4399    if *status != reqwest::StatusCode::CONFLICT {
4400        return false;
4401    }
4402
4403    let Some(run_id) = run_id else {
4404        return false;
4405    };
4406    let Ok(body) = serde_json::from_str::<Value>(body) else {
4407        return false;
4408    };
4409
4410    body.get("recorded").and_then(Value::as_bool) == Some(false)
4411        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4412        && body.get("run_status").and_then(Value::as_str) == Some("failed")
4413        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4414        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4415        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4416}
4417
4418fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4419    let body: Value = serde_json::from_str(raw_body).ok()?;
4420    let reason = body.get("reason")?.as_str()?;
4421    if !matches!(
4422        reason,
4423        "missing_protocol_version"
4424            | "unsupported_protocol_version"
4425            | "missing_control_plane_version"
4426            | "unsupported_control_plane_version"
4427    ) {
4428        return None;
4429    }
4430
4431    Some(ProtocolFailure {
4432        status: status.as_u16(),
4433        reason: reason.to_string(),
4434        message: body
4435            .get("message")
4436            .or_else(|| body.get("error"))
4437            .and_then(Value::as_str)
4438            .unwrap_or("protocol version rejected")
4439            .to_string(),
4440        supported_version: body
4441            .get("supported_version")
4442            .and_then(Value::as_str)
4443            .map(str::to_string),
4444        requested_version: body
4445            .get("requested_version")
4446            .and_then(Value::as_str)
4447            .map(str::to_string),
4448        body,
4449    })
4450}
4451
4452fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4453    timeout
4454        .as_secs()
4455        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4456        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4457}
4458
4459fn worker_operation_is_retryable(error: &Error) -> bool {
4460    if worker_poll_capacity_retry_after(error).is_some()
4461        || worker_storage_admission_body(error).is_some()
4462        || worker_operation_is_explicitly_non_retryable(error)
4463    {
4464        return false;
4465    }
4466
4467    match error {
4468        Error::Transport(error) => {
4469            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4470        }
4471        Error::Http { status, .. } => {
4472            matches!(
4473                *status,
4474                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4475            ) || status.is_server_error()
4476        }
4477        _ => false,
4478    }
4479}
4480
4481fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4482    let body: Value = match error {
4483        Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4484        Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4485        _ => return None,
4486    };
4487    matches!(
4488        body.get("reason").and_then(Value::as_str),
4489        Some("storage_pressure" | "storage_admission_unavailable")
4490    )
4491    .then_some(body)
4492}
4493
4494fn worker_storage_admission_retry_after(
4495    error: &Error,
4496    poll_request_id: Option<&str>,
4497) -> Option<Duration> {
4498    let Error::Http { status, .. } = error else {
4499        return None;
4500    };
4501    let body = worker_storage_admission_body(error)?;
4502    let delay = body.get("retry_after_seconds")?.as_u64()?;
4503    if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4504        || delay == 0
4505        || body.get("retryable") != Some(&Value::Bool(true))
4506        || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4507        || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4508        || body
4509            .get("request_admitted")
4510            .is_some_and(|admitted| admitted != &Value::Bool(false))
4511    {
4512        return None;
4513    }
4514    match poll_request_id {
4515        Some(id) => {
4516            if id.is_empty()
4517                || body.get("task") != Some(&Value::Null)
4518                || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4519                || body.get("poll_status") != body.get("reason")
4520                || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4521                || body.get("claim_admitted") != Some(&Value::Bool(false))
4522            {
4523                return None;
4524            }
4525        }
4526        None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4527        None => {}
4528    }
4529    Some(Duration::from_secs(delay))
4530}
4531
4532fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4533    let Error::Http { body, .. } = error else {
4534        return false;
4535    };
4536
4537    serde_json::from_str::<Value>(body)
4538        .ok()
4539        .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4540        == Some(false)
4541}
4542
4543fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4544    let Error::Http { status, body } = error else {
4545        return None;
4546    };
4547    if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4548        return None;
4549    }
4550
4551    let body = serde_json::from_str::<Value>(body).ok()?;
4552    let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4553        == Some("long_poll_capacity_exhausted")
4554        || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4555    if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4556        return None;
4557    }
4558
4559    Some(Duration::from_secs(
4560        body.get("retry_after_seconds")
4561            .and_then(Value::as_u64)
4562            .unwrap_or_default(),
4563    ))
4564}
4565
4566fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4567    let exponent = retry.saturating_sub(1).min(31) as u32;
4568    policy
4569        .initial_backoff
4570        .saturating_mul(1_u32 << exponent)
4571        .min(policy.max_backoff)
4572}
4573
4574#[derive(Debug)]
4575pub struct ClientBuilder {
4576    base_url: String,
4577    token: Option<String>,
4578    control_token: Option<String>,
4579    worker_token: Option<String>,
4580    namespace: String,
4581    timeout: Duration,
4582    max_external_payload_bytes: usize,
4583}
4584
4585impl ClientBuilder {
4586    pub fn token(mut self, token: Option<String>) -> Self {
4587        self.token = token;
4588        self
4589    }
4590
4591    pub fn control_token(mut self, token: Option<String>) -> Self {
4592        self.control_token = token;
4593        self
4594    }
4595
4596    pub fn worker_token(mut self, token: Option<String>) -> Self {
4597        self.worker_token = token;
4598        self
4599    }
4600
4601    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4602        self.namespace = namespace.into();
4603        self
4604    }
4605
4606    pub fn timeout(mut self, timeout: Duration) -> Self {
4607        self.timeout = timeout;
4608        self
4609    }
4610
4611    /// Maximum unique external-payload bytes fetched per response (default 64 MiB).
4612    /// References are fetched only from this client's authenticated runtime.
4613    pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
4614        self.max_external_payload_bytes = bytes;
4615        self
4616    }
4617
4618    pub fn build(self) -> Result<Client> {
4619        let base_url = self.base_url.trim_end_matches('/').to_string();
4620        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4621            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4622            .unwrap_or_else(|_| base_url.ends_with("/api"));
4623
4624        if has_sdk_api_suffix {
4625            return Err(Error::InvalidBaseUrl);
4626        }
4627
4628        Ok(Client {
4629            http: reqwest::Client::builder()
4630                .timeout(self.timeout)
4631                .redirect(reqwest::redirect::Policy::none())
4632                .build()?,
4633            base_url,
4634            token: self.token,
4635            control_token: self.control_token,
4636            worker_token: self.worker_token,
4637            namespace: self.namespace,
4638            max_external_payload_bytes: self.max_external_payload_bytes,
4639            worker_storage_admission: None,
4640        })
4641    }
4642}
4643
4644#[derive(Clone, Debug)]
4645pub struct WorkflowHandle {
4646    client: Client,
4647    pub workflow_id: String,
4648    pub run_id: Option<String>,
4649    pub workflow_type: String,
4650}
4651
4652impl WorkflowHandle {
4653    /// Describe whichever run is current for this stable workflow instance.
4654    pub async fn describe(&self) -> Result<WorkflowDescription> {
4655        self.client.describe_workflow(&self.workflow_id).await
4656    }
4657
4658    /// Describe the run identity originally selected by this handle.
4659    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4660        let run_id = self.run_id.as_deref().ok_or_else(|| {
4661            Error::Codec("run_id is required for selected-run description".to_string())
4662        })?;
4663        self.client
4664            .describe_workflow_run(&self.workflow_id, run_id)
4665            .await
4666    }
4667
4668    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4669        self.client
4670            .signal_workflow(&self.workflow_id, signal_name, input)
4671            .await
4672    }
4673
4674    pub async fn append_message<T: Serialize>(
4675        &self,
4676        stream_name: &str,
4677        message_id: &str,
4678        input: T,
4679    ) -> Result<Value> {
4680        self.client
4681            .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4682            .await
4683    }
4684
4685    /// Signal only if this handle's selected run is still current.
4686    pub async fn signal_selected_run<T: Serialize>(
4687        &self,
4688        signal_name: &str,
4689        input: T,
4690    ) -> Result<Value> {
4691        let run_id = self.run_id.as_deref().ok_or_else(|| {
4692            Error::Codec("run_id is required for selected-run signaling".to_string())
4693        })?;
4694        self.client
4695            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4696            .await
4697    }
4698
4699    /// Request cooperative cancellation of whichever run is current.
4700    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4701        self.client
4702            .cancel_workflow(&self.workflow_id, options)
4703            .await
4704    }
4705
4706    /// Request cancellation only if this handle's selected run is still current.
4707    pub async fn cancel_selected_run(
4708        &self,
4709        options: WorkflowCommandOptions,
4710    ) -> Result<WorkflowCommandResult> {
4711        let run_id = self.run_id.as_deref().ok_or_else(|| {
4712            Error::Codec("run_id is required for selected-run cancellation".to_string())
4713        })?;
4714        self.client
4715            .cancel_workflow_run(&self.workflow_id, run_id, options)
4716            .await
4717    }
4718
4719    /// Forcefully terminate whichever run is current.
4720    pub async fn terminate(
4721        &self,
4722        options: WorkflowCommandOptions,
4723    ) -> Result<WorkflowCommandResult> {
4724        self.client
4725            .terminate_workflow(&self.workflow_id, options)
4726            .await
4727    }
4728
4729    /// Terminate only if this handle's selected run is still current.
4730    pub async fn terminate_selected_run(
4731        &self,
4732        options: WorkflowCommandOptions,
4733    ) -> Result<WorkflowCommandResult> {
4734        let run_id = self.run_id.as_deref().ok_or_else(|| {
4735            Error::Codec("run_id is required for selected-run termination".to_string())
4736        })?;
4737        self.client
4738            .terminate_workflow_run(&self.workflow_id, run_id, options)
4739            .await
4740    }
4741
4742    /// Execute a named, read-only query against this workflow.
4743    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4744        self.client
4745            .query_workflow(&self.workflow_id, query_name, input)
4746            .await
4747    }
4748
4749    pub async fn query_avro_value<T: Serialize>(
4750        &self,
4751        query_name: &str,
4752        input: T,
4753    ) -> Result<AvroValue> {
4754        self.client
4755            .query_workflow_avro_value(&self.workflow_id, query_name, input)
4756            .await
4757    }
4758
4759    pub async fn update<T: Serialize>(
4760        &self,
4761        update_name: &str,
4762        input: T,
4763        request_id: Option<&str>,
4764    ) -> Result<Value> {
4765        self.client
4766            .update_workflow(&self.workflow_id, update_name, input, request_id)
4767            .await
4768    }
4769
4770    pub async fn update_avro_value<T: Serialize>(
4771        &self,
4772        update_name: &str,
4773        input: T,
4774        request_id: Option<&str>,
4775    ) -> Result<AvroValue> {
4776        self.client
4777            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4778            .await
4779    }
4780
4781    /// Query only if this handle's selected run is still current.
4782    pub async fn query_selected_run<T: Serialize>(
4783        &self,
4784        query_name: &str,
4785        input: T,
4786    ) -> Result<Value> {
4787        let run_id = self
4788            .run_id
4789            .as_deref()
4790            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4791        self.client
4792            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4793            .await
4794    }
4795
4796    /// Await the final terminal outcome of the current continue-as-new chain.
4797    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4798        self.result_target(options, None).await
4799    }
4800
4801    /// Await the final result without projecting Avro bytes through JSON.
4802    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4803        self.result_avro_value_target(options, None).await
4804    }
4805
4806    /// Await the final result and decode it into a Serde application type.
4807    pub async fn result_typed<T: DeserializeOwned>(
4808        &self,
4809        options: WorkflowResultOptions,
4810    ) -> Result<T> {
4811        let result = self.result_avro_value(options).await?;
4812        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4813    }
4814
4815    /// Await only the run identity originally selected by this handle.
4816    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4817        let run_id = self.run_id.as_deref().ok_or_else(|| {
4818            Error::Codec("run_id is required for selected-run result".to_string())
4819        })?;
4820        self.result_target(options, Some(run_id)).await
4821    }
4822
4823    /// Await the selected run's result on the lossless Avro Value surface.
4824    pub async fn result_selected_run_avro_value(
4825        &self,
4826        options: WorkflowResultOptions,
4827    ) -> Result<AvroValue> {
4828        let run_id = self.run_id.as_deref().ok_or_else(|| {
4829            Error::Codec("run_id is required for selected-run result".to_string())
4830        })?;
4831        self.result_avro_value_target(options, Some(run_id)).await
4832    }
4833
4834    /// Await the selected run and decode its result into a Serde type.
4835    pub async fn result_selected_run_typed<T: DeserializeOwned>(
4836        &self,
4837        options: WorkflowResultOptions,
4838    ) -> Result<T> {
4839        let result = self.result_selected_run_avro_value(options).await?;
4840        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4841    }
4842
4843    async fn result_avro_value_target(
4844        &self,
4845        options: WorkflowResultOptions,
4846        selected_run_id: Option<&str>,
4847    ) -> Result<AvroValue> {
4848        let started = Instant::now();
4849
4850        loop {
4851            let description = match selected_run_id {
4852                Some(run_id) => {
4853                    self.client
4854                        .describe_workflow_run(&self.workflow_id, run_id)
4855                        .await?
4856                }
4857                None => self.describe().await?,
4858            };
4859            if description.is_completed() {
4860                return description.output_avro_value.ok_or_else(|| {
4861                    Error::Codec(
4862                        "missing_payload_envelope: typed workflow result requires output_envelope"
4863                            .to_string(),
4864                    )
4865                });
4866            }
4867            if description.is_terminal() {
4868                let outcome =
4869                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4870                return Err(match outcome.kind {
4871                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4872                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4873                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4874                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4875                });
4876            }
4877            if started.elapsed() >= options.timeout {
4878                return Err(Error::Timeout);
4879            }
4880            tokio::time::sleep(options.poll_interval).await;
4881        }
4882    }
4883
4884    async fn result_target(
4885        &self,
4886        options: WorkflowResultOptions,
4887        selected_run_id: Option<&str>,
4888    ) -> Result<Value> {
4889        let started = Instant::now();
4890
4891        loop {
4892            let description = match selected_run_id {
4893                Some(run_id) => {
4894                    self.client
4895                        .describe_workflow_run(&self.workflow_id, run_id)
4896                        .await?
4897                }
4898                None => self.describe().await?,
4899            };
4900            if description.is_completed() {
4901                return Ok(description.output.unwrap_or(Value::Null));
4902            }
4903
4904            if description.is_terminal() {
4905                let outcome =
4906                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4907                return Err(match outcome.kind {
4908                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4909                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4910                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4911                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4912                });
4913            }
4914
4915            if started.elapsed() >= options.timeout {
4916                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4917                    kind: WorkflowTerminalKind::TimedOut,
4918                    workflow_id: description
4919                        .workflow_id
4920                        .clone()
4921                        .unwrap_or_else(|| self.workflow_id.clone()),
4922                    run_id: description
4923                        .run_id
4924                        .clone()
4925                        .or_else(|| selected_run_id.map(str::to_string)),
4926                    reason: "result_wait_timeout".to_string(),
4927                    failure_category: Some("client_timeout".to_string()),
4928                    failure_id: None,
4929                    exception_type: None,
4930                    exception_class: None,
4931                    non_retryable: None,
4932                    message: Some(format!(
4933                        "workflow result was not terminal within {:?}",
4934                        options.timeout
4935                    )),
4936                    exception: None,
4937                    raw: description.raw_value(),
4938                }));
4939            }
4940
4941            tokio::time::sleep(options.poll_interval).await;
4942        }
4943    }
4944}
4945
4946#[derive(Clone, Copy, Debug)]
4947pub struct WorkflowResultOptions {
4948    pub poll_interval: Duration,
4949    pub timeout: Duration,
4950}
4951
4952impl Default for WorkflowResultOptions {
4953    fn default() -> Self {
4954        Self {
4955            poll_interval: Duration::from_millis(500),
4956            timeout: Duration::from_secs(30),
4957        }
4958    }
4959}
4960
4961#[derive(Clone, Debug, Deserialize)]
4962pub struct WorkflowDescription {
4963    pub workflow_id: Option<String>,
4964    pub run_id: Option<String>,
4965    pub workflow_type: Option<String>,
4966    pub status: Option<String>,
4967    #[serde(default)]
4968    pub closed_reason: Option<String>,
4969    #[serde(default)]
4970    pub error: Option<String>,
4971    #[serde(default)]
4972    pub failure: Option<Value>,
4973    #[serde(default)]
4974    pub exception: Option<Value>,
4975    #[serde(default)]
4976    pub failures: Vec<Value>,
4977    #[serde(default)]
4978    pub output: Option<Value>,
4979    #[serde(default)]
4980    pub output_envelope: Option<Value>,
4981    #[serde(skip)]
4982    pub output_avro_value: Option<AvroValue>,
4983    #[serde(flatten)]
4984    pub raw: HashMap<String, Value>,
4985}
4986
4987/// Lifecycle and backlog metadata for one run-scoped Workflow Stream.
4988#[derive(Clone, Debug, Deserialize)]
4989pub struct WorkflowStreamDescription {
4990    pub stream_name: String,
4991    pub status: String,
4992    pub last_offset: i64,
4993    pub total_items: u64,
4994    pub pending_items: u64,
4995    #[serde(default)]
4996    pub opened_at: Option<String>,
4997    #[serde(default)]
4998    pub last_appended_at: Option<String>,
4999    #[serde(default)]
5000    pub closed_at: Option<String>,
5001    #[serde(default)]
5002    pub error_reason: Option<String>,
5003    #[serde(default)]
5004    pub retention_seconds: Option<u64>,
5005    #[serde(flatten)]
5006    pub raw: HashMap<String, Value>,
5007}
5008
5009impl WorkflowStreamDescription {
5010    pub fn is_terminal(&self) -> bool {
5011        matches!(self.status.as_str(), "closed" | "errored")
5012    }
5013}
5014
5015/// One item for direct or replay-safe append.
5016#[derive(Clone, Debug, Default)]
5017pub struct WorkflowStreamAppendItem {
5018    pub payload_envelope: Option<Value>,
5019    pub payload_reference: Option<String>,
5020    pub item_type: Option<String>,
5021    pub content_type: Option<String>,
5022    pub idempotency_key: Option<String>,
5023}
5024
5025impl WorkflowStreamAppendItem {
5026    /// Encode an inline payload with the SDK's fixed Avro Value envelope.
5027    pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5028        let value = AvroValue::from_serialize(&payload)?;
5029        Ok(Self {
5030            payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5031            ..Self::default()
5032        })
5033    }
5034
5035    /// Preserve an external payload URI as an opaque service-contract reference.
5036    pub fn from_reference(reference: impl Into<String>) -> Self {
5037        Self {
5038            payload_reference: Some(reference.into()),
5039            ..Self::default()
5040        }
5041    }
5042
5043    pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5044        self.item_type = Some(item_type.into());
5045        self
5046    }
5047
5048    pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5049        self.content_type = Some(content_type.into());
5050        self
5051    }
5052
5053    pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5054        self.idempotency_key = Some(idempotency_key.into());
5055        self
5056    }
5057
5058    fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5059        let mut item = serde_json::Map::new();
5060        if let Some(payload) = &self.payload_envelope {
5061            item.insert("payload".to_string(), payload.clone());
5062            item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5063        }
5064        if let Some(reference) = &self.payload_reference {
5065            item.insert("payload_reference".to_string(), json!(reference));
5066        }
5067        if let Some(item_type) = &self.item_type {
5068            item.insert("item_type".to_string(), json!(item_type));
5069        }
5070        if let Some(content_type) = &self.content_type {
5071            item.insert("content_type".to_string(), json!(content_type));
5072        }
5073        if let Some(key) = derived_idempotency_key
5074            .as_ref()
5075            .or(self.idempotency_key.as_ref())
5076        {
5077            item.insert("idempotency_key".to_string(), json!(key));
5078        }
5079        Value::Object(item)
5080    }
5081}
5082
5083/// One durable item at its stable zero-based offset.
5084#[derive(Clone, Debug)]
5085pub struct WorkflowStreamItem {
5086    pub offset: u64,
5087    pub payload: Option<Value>,
5088    pub payload_envelope: Option<Value>,
5089    pub payload_reference: Option<String>,
5090    pub payload_codec: Option<String>,
5091    pub idempotency_key: Option<String>,
5092    pub item_type: Option<String>,
5093    pub content_type: Option<String>,
5094    pub origin: Option<String>,
5095    pub origin_reference: Option<String>,
5096    pub emitted_at: Option<String>,
5097    pub raw: Value,
5098}
5099
5100/// One bounded at-least-once subscription page.
5101#[derive(Clone, Debug)]
5102pub struct WorkflowStreamPage {
5103    pub stream: WorkflowStreamDescription,
5104    pub items: Vec<WorkflowStreamItem>,
5105    pub next_offset: u64,
5106    pub terminal: bool,
5107}
5108
5109/// Durable acceptance and deduplication outcome for an append request.
5110#[derive(Clone, Debug)]
5111pub struct WorkflowStreamAppendResult {
5112    pub stream: WorkflowStreamDescription,
5113    pub accepted_offsets: Vec<u64>,
5114    pub accepted: u64,
5115    pub deduped: u64,
5116}
5117
5118#[derive(Deserialize)]
5119struct WorkflowStreamListResponse {
5120    #[serde(default)]
5121    streams: Vec<WorkflowStreamDescription>,
5122}
5123
5124#[derive(Deserialize)]
5125struct WorkflowStreamDescriptionResponse {
5126    stream: WorkflowStreamDescription,
5127}
5128
5129#[derive(Deserialize)]
5130struct WorkflowStreamPageResponse {
5131    stream: WorkflowStreamDescription,
5132    #[serde(default)]
5133    items: Vec<Value>,
5134    next_offset: u64,
5135    terminal: bool,
5136}
5137
5138#[derive(Deserialize)]
5139struct WorkflowStreamAppendResponse {
5140    stream: WorkflowStreamDescription,
5141    #[serde(default)]
5142    accepted_offsets: Vec<u64>,
5143    accepted: u64,
5144    deduped: u64,
5145}
5146
5147impl WorkflowDescription {
5148    pub fn is_completed(&self) -> bool {
5149        matches!(self.status.as_deref(), Some("completed" | "Completed"))
5150    }
5151
5152    pub fn is_terminal(&self) -> bool {
5153        matches!(
5154            self.status.as_deref(),
5155            Some(
5156                "completed"
5157                    | "Completed"
5158                    | "failed"
5159                    | "Failed"
5160                    | "cancelled"
5161                    | "Cancelled"
5162                    | "terminated"
5163                    | "Terminated"
5164                    | "timed_out"
5165                    | "TimedOut",
5166            )
5167        )
5168    }
5169
5170    fn decode_payloads(&mut self) -> Result<()> {
5171        if let Some(envelope) = &self.output_envelope {
5172            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5173            self.output = Some(value.clone().into_json()?);
5174            self.output_avro_value = Some(value);
5175        }
5176
5177        Ok(())
5178    }
5179
5180    fn raw_value(&self) -> Value {
5181        let mut data = self.raw.clone();
5182        data.insert(
5183            "workflow_id".to_string(),
5184            self.workflow_id
5185                .clone()
5186                .map(Value::String)
5187                .unwrap_or(Value::Null),
5188        );
5189        data.insert(
5190            "run_id".to_string(),
5191            self.run_id
5192                .clone()
5193                .map(Value::String)
5194                .unwrap_or(Value::Null),
5195        );
5196        data.insert(
5197            "workflow_type".to_string(),
5198            self.workflow_type
5199                .clone()
5200                .map(Value::String)
5201                .unwrap_or(Value::Null),
5202        );
5203        data.insert(
5204            "status".to_string(),
5205            self.status
5206                .clone()
5207                .map(Value::String)
5208                .unwrap_or(Value::Null),
5209        );
5210        data.insert(
5211            "closed_reason".to_string(),
5212            self.closed_reason
5213                .clone()
5214                .map(Value::String)
5215                .unwrap_or(Value::Null),
5216        );
5217        if let Some(failure) = &self.failure {
5218            data.insert("failure".to_string(), failure.clone());
5219        }
5220        if let Some(exception) = &self.exception {
5221            data.insert("exception".to_string(), exception.clone());
5222        }
5223        Value::Object(data.into_iter().collect())
5224    }
5225}
5226
5227fn workflow_terminal_outcome(
5228    description: &WorkflowDescription,
5229    workflow_id: &str,
5230    run_id: Option<&str>,
5231) -> WorkflowTerminalOutcome {
5232    let terminal_kind = description
5233        .closed_reason
5234        .as_deref()
5235        .or(description.status.as_deref())
5236        .unwrap_or("failed")
5237        .to_ascii_lowercase();
5238    let kind = match terminal_kind.as_str() {
5239        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5240        "terminated" => WorkflowTerminalKind::Terminated,
5241        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5242        _ => WorkflowTerminalKind::Failed,
5243    };
5244    let default_reason = match kind {
5245        WorkflowTerminalKind::Failed => "workflow_failed",
5246        WorkflowTerminalKind::Cancelled => "cancelled",
5247        WorkflowTerminalKind::Terminated => "terminated",
5248        WorkflowTerminalKind::TimedOut => "timed_out",
5249    };
5250    let failure = description
5251        .failure
5252        .as_ref()
5253        .filter(|value| value.is_object());
5254    let nested_failure = failure
5255        .and_then(|value| value.get("failures"))
5256        .and_then(Value::as_array)
5257        .and_then(|failures| failures.last())
5258        .or_else(|| description.failures.last());
5259    let exception = description
5260        .exception
5261        .clone()
5262        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5263        .or_else(|| {
5264            nested_failure
5265                .and_then(|value| value.get("exception_payload"))
5266                .cloned()
5267        });
5268    let string_field = |name: &str| {
5269        failure
5270            .and_then(|value| value.get(name))
5271            .and_then(Value::as_str)
5272            .or_else(|| {
5273                nested_failure
5274                    .and_then(|value| value.get(name))
5275                    .and_then(Value::as_str)
5276            })
5277            .map(str::to_string)
5278    };
5279    let exception_field = |name: &str| {
5280        exception
5281            .as_ref()
5282            .and_then(|value| value.get(name))
5283            .and_then(Value::as_str)
5284            .map(str::to_string)
5285    };
5286    let message = description
5287        .error
5288        .clone()
5289        .or_else(|| string_field("message"))
5290        .or_else(|| exception_field("message"));
5291    let reason = description
5292        .raw
5293        .get("reason")
5294        .and_then(Value::as_str)
5295        .map(str::to_string)
5296        .or_else(|| {
5297            failure
5298                .and_then(|value| value.get("reason"))
5299                .and_then(Value::as_str)
5300                .map(str::to_string)
5301        })
5302        .or_else(|| description.closed_reason.clone())
5303        .unwrap_or_else(|| default_reason.to_string());
5304    let failure_id = string_field("failure_id").or_else(|| {
5305        nested_failure
5306            .and_then(|value| value.get("id"))
5307            .and_then(Value::as_str)
5308            .map(str::to_string)
5309    });
5310
5311    WorkflowTerminalOutcome {
5312        kind,
5313        workflow_id: description
5314            .workflow_id
5315            .clone()
5316            .unwrap_or_else(|| workflow_id.to_string()),
5317        run_id: description
5318            .run_id
5319            .clone()
5320            .or_else(|| run_id.map(str::to_string)),
5321        reason,
5322        failure_category: string_field("failure_category")
5323            .or_else(|| Some(default_reason.to_string())),
5324        failure_id,
5325        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5326        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5327        non_retryable: failure
5328            .and_then(|value| value.get("non_retryable"))
5329            .and_then(Value::as_bool)
5330            .or_else(|| {
5331                nested_failure
5332                    .and_then(|value| value.get("non_retryable"))
5333                    .and_then(Value::as_bool)
5334            }),
5335        message,
5336        exception,
5337        raw: description.raw_value(),
5338    }
5339}
5340
5341#[derive(Clone, Debug, Deserialize)]
5342pub struct RegisterWorkerResponse {
5343    pub worker_id: String,
5344    pub registered: bool,
5345    #[serde(default)]
5346    pub heartbeat_interval_seconds: Option<u64>,
5347    #[serde(default)]
5348    pub protocol_version: Option<String>,
5349    #[serde(default)]
5350    pub server_capabilities: Option<Value>,
5351}
5352
5353/// Result of gracefully removing a worker-plane registration.
5354#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5355pub struct WorkerDeregistrationEnvelope {
5356    pub worker_id: String,
5357    pub outcome: String,
5358    pub recovered_workflow_task_count: u64,
5359}
5360
5361#[derive(Clone, Debug, Deserialize)]
5362pub struct PollWorkflowTaskResponse {
5363    #[serde(default)]
5364    pub task: Option<WorkflowTask>,
5365    #[serde(default)]
5366    pub poll_status: Option<String>,
5367    #[serde(default)]
5368    pub reason: Option<String>,
5369    #[serde(default)]
5370    pub protocol_version: Option<String>,
5371    #[serde(default)]
5372    pub server_capabilities: Option<Value>,
5373}
5374
5375impl PollWorkflowTaskResponse {
5376    /// Classify this response without parsing server display text.
5377    pub fn outcome(&self) -> WorkerPollOutcome {
5378        worker_poll_outcome(
5379            self.task.is_some(),
5380            self.poll_status.as_deref(),
5381            self.reason.as_deref(),
5382        )
5383    }
5384}
5385
5386fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5387    let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5388        return false;
5389    };
5390    let supported = capabilities
5391        .get("workflow_memo_updates")
5392        .and_then(Value::as_object)
5393        .and_then(|memo| memo.get("supported"))
5394        .and_then(Value::as_bool)
5395        == Some(true);
5396    let command_advertised = capabilities
5397        .get("supported_workflow_task_commands")
5398        .and_then(Value::as_array)
5399        .is_some_and(|commands| {
5400            commands
5401                .iter()
5402                .any(|command| command.as_str() == Some("upsert_memo"))
5403        });
5404    supported && command_advertised
5405}
5406
5407fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5408    commands
5409        .iter()
5410        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5411}
5412
5413#[derive(Clone, Debug, Deserialize)]
5414pub struct PollActivityTaskResponse {
5415    #[serde(default)]
5416    pub task: Option<ActivityTask>,
5417    #[serde(default)]
5418    pub poll_status: Option<String>,
5419    #[serde(default)]
5420    pub reason: Option<String>,
5421}
5422
5423impl PollActivityTaskResponse {
5424    /// Classify this response without parsing server display text.
5425    pub fn outcome(&self) -> WorkerPollOutcome {
5426        worker_poll_outcome(
5427            self.task.is_some(),
5428            self.poll_status.as_deref(),
5429            self.reason.as_deref(),
5430        )
5431    }
5432}
5433
5434#[derive(Clone, Debug, Deserialize)]
5435pub struct PollQueryTaskResponse {
5436    #[serde(default)]
5437    pub task: Option<QueryTask>,
5438    #[serde(default)]
5439    pub poll_status: Option<String>,
5440    #[serde(default)]
5441    pub reason: Option<String>,
5442}
5443
5444impl PollQueryTaskResponse {
5445    /// Classify this response without parsing server display text.
5446    pub fn outcome(&self) -> WorkerPollOutcome {
5447        worker_poll_outcome(
5448            self.task.is_some(),
5449            self.poll_status.as_deref(),
5450            self.reason.as_deref(),
5451        )
5452    }
5453}
5454
5455/// Stable classification for worker poll responses.
5456#[derive(Clone, Debug, PartialEq, Eq)]
5457pub enum WorkerPollOutcome {
5458    /// A task was leased and is available on the response.
5459    Task,
5460    /// No task was leased, but the worker should continue polling.
5461    Idle {
5462        poll_status: Option<String>,
5463        reason: Option<String>,
5464    },
5465    /// The server asked this worker to stop claiming new work.
5466    Stop {
5467        poll_status: Option<String>,
5468        reason: Option<String>,
5469    },
5470}
5471
5472impl WorkerPollOutcome {
5473    pub fn should_stop(&self) -> bool {
5474        matches!(self, Self::Stop { .. })
5475    }
5476}
5477
5478fn worker_poll_outcome(
5479    has_task: bool,
5480    poll_status: Option<&str>,
5481    reason: Option<&str>,
5482) -> WorkerPollOutcome {
5483    if worker_poll_is_stop(poll_status, reason) {
5484        return WorkerPollOutcome::Stop {
5485            poll_status: poll_status.map(str::to_string),
5486            reason: reason.map(str::to_string),
5487        };
5488    }
5489
5490    if has_task {
5491        WorkerPollOutcome::Task
5492    } else {
5493        WorkerPollOutcome::Idle {
5494            poll_status: poll_status.map(str::to_string),
5495            reason: reason.map(str::to_string),
5496        }
5497    }
5498}
5499
5500/// An ephemeral server-routed query task.
5501#[derive(Clone, Debug, Deserialize)]
5502pub struct QueryTask {
5503    pub query_task_id: String,
5504    #[serde(default = "default_workflow_task_attempt")]
5505    pub query_task_attempt: u64,
5506    #[serde(default)]
5507    pub lease_owner: Option<String>,
5508    #[serde(default)]
5509    pub workflow_id: Option<String>,
5510    #[serde(default)]
5511    pub run_id: Option<String>,
5512    pub workflow_type: String,
5513    pub query_name: String,
5514    #[serde(
5515        default = "missing_task_payload_codec",
5516        deserialize_with = "deserialize_task_payload_codec"
5517    )]
5518    pub payload_codec: String,
5519    #[serde(default)]
5520    pub workflow_arguments: Option<Value>,
5521    #[serde(default)]
5522    pub query_arguments: Option<Value>,
5523    #[serde(default)]
5524    pub history_events: Vec<HistoryEvent>,
5525    #[serde(default)]
5526    pub history_export: Option<Value>,
5527    #[serde(default)]
5528    pub run_status: Option<String>,
5529}
5530
5531#[derive(Clone, Debug, Deserialize)]
5532pub struct WorkflowTask {
5533    pub task_id: String,
5534    #[serde(default)]
5535    pub workflow_command_id: Option<String>,
5536    #[serde(default)]
5537    pub workflow_id: Option<String>,
5538    #[serde(default)]
5539    pub run_id: Option<String>,
5540    pub workflow_type: String,
5541    #[serde(default)]
5542    pub cancel_requested: bool,
5543    #[serde(
5544        default = "missing_task_payload_codec",
5545        deserialize_with = "deserialize_task_payload_codec"
5546    )]
5547    pub payload_codec: String,
5548    #[serde(default)]
5549    pub arguments: Option<Value>,
5550    #[serde(default)]
5551    pub history_events: Vec<HistoryEvent>,
5552    #[serde(default)]
5553    pub total_history_events: Option<u64>,
5554    #[serde(default)]
5555    pub history_size_bytes: Option<u64>,
5556    #[serde(default)]
5557    pub continue_as_new_recommended: Option<bool>,
5558    #[serde(default)]
5559    pub history_budget_pressure: Option<String>,
5560    #[serde(default)]
5561    pub next_history_page_token: Option<String>,
5562    #[serde(default = "default_workflow_task_attempt")]
5563    pub workflow_task_attempt: u64,
5564    #[serde(default)]
5565    pub workflow_signal_id: Option<String>,
5566    #[serde(default)]
5567    pub signal_name: Option<String>,
5568    #[serde(default)]
5569    pub signal_arguments: Option<Value>,
5570    #[serde(default)]
5571    pub workflow_update_id: Option<String>,
5572    #[serde(default)]
5573    pub update_name: Option<String>,
5574    #[serde(default)]
5575    pub lease_owner: Option<String>,
5576}
5577
5578impl WorkflowTask {
5579    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5580        self.history_events.extend(page.history_events);
5581
5582        if page.total_history_events.is_some() {
5583            self.total_history_events = page.total_history_events;
5584        }
5585
5586        self.next_history_page_token = page
5587            .next_history_page_token
5588            .filter(|token| !token.is_empty());
5589    }
5590}
5591
5592#[derive(Clone, Debug, Deserialize)]
5593struct WorkflowTaskHistoryPage {
5594    #[serde(default)]
5595    history_events: Vec<HistoryEvent>,
5596    #[serde(default)]
5597    total_history_events: Option<u64>,
5598    #[serde(default)]
5599    next_history_page_token: Option<String>,
5600}
5601
5602#[derive(Clone, Debug, Deserialize)]
5603pub struct ActivityTask {
5604    pub task_id: String,
5605    #[serde(default)]
5606    pub activity_attempt_id: Option<String>,
5607    #[serde(default)]
5608    pub attempt_id: Option<String>,
5609    pub activity_type: String,
5610    #[serde(
5611        default = "missing_task_payload_codec",
5612        deserialize_with = "deserialize_task_payload_codec"
5613    )]
5614    pub payload_codec: String,
5615    #[serde(default)]
5616    pub arguments: Option<Value>,
5617    #[serde(default = "default_attempt_number")]
5618    pub attempt_number: u64,
5619    #[serde(default)]
5620    pub lease_owner: Option<String>,
5621}
5622
5623#[derive(Clone, Debug, Deserialize)]
5624pub struct HistoryEvent {
5625    #[serde(alias = "type")]
5626    pub event_type: String,
5627    #[serde(default)]
5628    pub payload: Value,
5629    #[serde(flatten)]
5630    pub raw: HashMap<String, Value>,
5631}
5632
5633/// One decoded signal in the committed workflow-history snapshot.
5634#[derive(Clone, Debug, PartialEq)]
5635pub struct QuerySignal {
5636    pub id: Option<String>,
5637    pub name: String,
5638    pub arguments: Vec<Value>,
5639    avro_arguments: Vec<AvroValue>,
5640    pub workflow_sequence: Option<u64>,
5641}
5642
5643impl QuerySignal {
5644    /// Lossless fixed Avro Value arguments for this committed signal.
5645    pub fn arguments_avro_value(&self) -> &[AvroValue] {
5646        &self.avro_arguments
5647    }
5648}
5649
5650/// Immutable state supplied to a registered query handler.
5651///
5652/// This context intentionally exposes no activity, signal-wait, or command
5653/// APIs. Query handlers inspect committed history and return a value; query
5654/// completion does not append an event or advance deterministic execution.
5655#[derive(Clone, Debug)]
5656pub struct QueryContext {
5657    pub workflow_id: Option<String>,
5658    pub run_id: Option<String>,
5659    pub workflow_type: String,
5660    pub run_status: Option<String>,
5661    workflow_input: Value,
5662    workflow_input_avro_value: AvroValue,
5663    history_events: Arc<Vec<HistoryEvent>>,
5664    signal_events: Arc<Vec<QuerySignal>>,
5665}
5666
5667impl QueryContext {
5668    /// The normalized argument list used to start the workflow.
5669    pub fn workflow_input(&self) -> &Value {
5670        &self.workflow_input
5671    }
5672
5673    /// The lossless fixed Avro Value argument list used to start the workflow.
5674    pub fn workflow_input_avro_value(&self) -> &AvroValue {
5675        &self.workflow_input_avro_value
5676    }
5677
5678    /// The immutable committed history used for this query snapshot.
5679    pub fn history_events(&self) -> &[HistoryEvent] {
5680        self.history_events.as_slice()
5681    }
5682
5683    /// All decoded signals in committed workflow order.
5684    pub fn signal_events(&self) -> &[QuerySignal] {
5685        self.signal_events.as_slice()
5686    }
5687
5688    /// Decoded argument lists for each committed signal with `signal_name`.
5689    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5690        self.signal_events
5691            .iter()
5692            .filter(|signal| signal.name == signal_name)
5693            .map(|signal| signal.arguments.clone())
5694            .collect()
5695    }
5696
5697    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
5698    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5699        self.signal_events
5700            .iter()
5701            .filter(|signal| signal.name == signal_name)
5702            .map(|signal| signal.avro_arguments.clone())
5703            .collect()
5704    }
5705}
5706
5707#[derive(Clone, Debug, Deserialize)]
5708pub struct ActivityHeartbeatResponse {
5709    #[serde(default)]
5710    pub cancel_requested: bool,
5711    #[serde(default)]
5712    pub heartbeat_recorded: bool,
5713    #[serde(default)]
5714    pub can_continue: Option<bool>,
5715    #[serde(default)]
5716    pub reason: Option<String>,
5717    #[serde(default)]
5718    pub run_closed_reason: Option<String>,
5719    #[serde(default)]
5720    pub run_closed_at: Option<String>,
5721    #[serde(default)]
5722    pub lease_expires_at: Option<String>,
5723    #[serde(default)]
5724    pub last_heartbeat_at: Option<String>,
5725}
5726
5727impl ActivityHeartbeatResponse {
5728    /// Whether the activity should stop instead of attempting completion.
5729    pub fn should_stop(&self) -> bool {
5730        self.cancel_requested || self.can_continue == Some(false)
5731    }
5732}
5733
5734fn missing_task_payload_codec() -> String {
5735    MISSING_TASK_PAYLOAD_CODEC.to_string()
5736}
5737
5738fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5739where
5740    D: Deserializer<'de>,
5741{
5742    Ok(match Value::deserialize(deserializer)? {
5743        Value::String(codec) => codec,
5744        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5745        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5746    })
5747}
5748
5749fn default_workflow_task_attempt() -> u64 {
5750    1
5751}
5752
5753fn default_attempt_number() -> u64 {
5754    1
5755}
5756
5757type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5758type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5759type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5760type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5761type ReplayedWorkflowHandler =
5762    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5763type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5764type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5765type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5766type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5767type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5768type ReplayedQueryHandler = Arc<
5769    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5770        + Send
5771        + Sync,
5772>;
5773type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5774
5775struct ReplayedWorkflowInvocation {
5776    future: WorkflowFuture,
5777    snapshot: WorkflowStateSnapshot,
5778}
5779
5780#[derive(Clone)]
5781struct RegisteredWorkflow {
5782    execute: WorkflowHandler,
5783    replay: Option<ReplayedWorkflowHandler>,
5784    state_type: Option<TypeId>,
5785}
5786
5787#[derive(Debug)]
5788struct WorkflowTaskDecision {
5789    commands: Vec<Value>,
5790    message_stream_cursors: Vec<Value>,
5791    message_stream_waits: Vec<Value>,
5792}
5793
5794impl WorkflowTaskDecision {
5795    fn without_message_streams(commands: Vec<Value>) -> Self {
5796        Self {
5797            commands,
5798            message_stream_cursors: Vec::new(),
5799            message_stream_waits: Vec::new(),
5800        }
5801    }
5802}
5803
5804#[derive(Clone)]
5805enum RegisteredQuery {
5806    Snapshot(QueryHandler),
5807    Replayed {
5808        state_type: TypeId,
5809        handler: ReplayedQueryHandler,
5810    },
5811}
5812
5813#[derive(Clone, Debug)]
5814pub struct WorkerHeartbeatObservation {
5815    pub worker_id: String,
5816    pub task_queue: String,
5817    pub acknowledged_at_unix_millis: u64,
5818    pub acknowledgement: Value,
5819}
5820
5821/// Bounded retry policy for worker poll acquisition and worker heartbeats.
5822///
5823/// Expected empty long polls and explicit Server capacity backpressure are
5824/// normal worker states. Capacity backpressure honors the advertised retry
5825/// delay without consuming this retry budget. Other transport failures,
5826/// retryable HTTP 408/429 responses, and server errors are retried with capped
5827/// exponential backoff. Authentication, protocol, codec, and handler failures
5828/// are never retried by the worker.
5829/// Explicit storage admission pauses also preserve the already-serialized worker
5830/// request without consuming this budget or rerunning its handler. The delay is
5831/// capped by `max_backoff`, and shutdown interrupts storage waits.
5832#[derive(Clone, Copy, Debug)]
5833pub struct WorkerRetryPolicy {
5834    /// Number of retries after the initial request fails.
5835    pub max_retries: usize,
5836    /// Delay before the first retry.
5837    pub initial_backoff: Duration,
5838    /// Maximum delay between retries.
5839    pub max_backoff: Duration,
5840}
5841
5842impl Default for WorkerRetryPolicy {
5843    fn default() -> Self {
5844        Self {
5845            max_retries: 5,
5846            initial_backoff: Duration::from_millis(100),
5847            max_backoff: Duration::from_secs(5),
5848        }
5849    }
5850}
5851
5852#[derive(Clone, Debug)]
5853struct WorkerStorageAdmission {
5854    policy: WorkerRetryPolicy,
5855    stop: Arc<AtomicBool>,
5856}
5857
5858struct StopWorkerOnDrop(Arc<AtomicBool>);
5859
5860impl Drop for StopWorkerOnDrop {
5861    fn drop(&mut self) {
5862        self.0.store(true, Ordering::SeqCst);
5863    }
5864}
5865
5866async fn wait_for_worker_stop(stop: &AtomicBool) {
5867    while !stop.load(Ordering::SeqCst) {
5868        tokio::time::sleep(Duration::from_millis(100)).await;
5869    }
5870}
5871
5872#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5873enum ManagedPollOutcome {
5874    Idle,
5875    Handled,
5876    Stop,
5877}
5878
5879#[derive(Clone)]
5880pub struct Worker {
5881    client: Client,
5882    worker_id: String,
5883    task_queue: String,
5884    workflows: HashMap<String, RegisteredWorkflow>,
5885    activities: HashMap<String, ActivityHandler>,
5886    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5887    updates: HashMap<String, HashMap<String, UpdateHandler>>,
5888    max_concurrent_workflow_tasks: usize,
5889    max_concurrent_activity_tasks: usize,
5890    poll_timeout: Duration,
5891    heartbeat_interval: Duration,
5892    retry_policy: WorkerRetryPolicy,
5893    heartbeat_observer: Option<WorkerHeartbeatObserver>,
5894}
5895
5896impl Worker {
5897    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5898        Self {
5899            client,
5900            worker_id: default_worker_id(),
5901            task_queue: task_queue.into(),
5902            workflows: HashMap::new(),
5903            activities: HashMap::new(),
5904            queries: HashMap::new(),
5905            updates: HashMap::new(),
5906            max_concurrent_workflow_tasks: 10,
5907            max_concurrent_activity_tasks: 10,
5908            poll_timeout: Duration::from_secs(30),
5909            heartbeat_interval: Duration::from_secs(60),
5910            retry_policy: WorkerRetryPolicy::default(),
5911            heartbeat_observer: None,
5912        }
5913    }
5914
5915    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5916        self.worker_id = worker_id.into();
5917        self
5918    }
5919
5920    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5921        self.poll_timeout = timeout;
5922        self
5923    }
5924
5925    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5926        self.heartbeat_interval = interval;
5927        self
5928    }
5929
5930    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
5931    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5932        self.retry_policy = policy;
5933        self
5934    }
5935
5936    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5937    where
5938        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5939    {
5940        self.heartbeat_observer = Some(Arc::new(observer));
5941        self
5942    }
5943
5944    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5945        self.max_concurrent_workflow_tasks = count.max(1);
5946        self
5947    }
5948
5949    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5950        self.max_concurrent_activity_tasks = count.max(1);
5951        self
5952    }
5953
5954    /// Register a workflow handler.
5955    ///
5956    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
5957    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
5958    /// while acquiring or decoding a worker task remain worker-operation
5959    /// failures and do not get converted into workflow outcomes.
5960    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5961    where
5962        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5963        Fut: Future<Output = Result<Value>> + Send + 'static,
5964    {
5965        let handler = Arc::new(handler);
5966        self.workflows.insert(
5967            workflow_type.into(),
5968            RegisteredWorkflow {
5969                execute: Arc::new(move |ctx, input| {
5970                    let handler = Arc::clone(&handler);
5971                    Box::pin(async move {
5972                        let result = handler(ctx, input.into_json()?).await?;
5973                        AvroValue::from_serialize(&result)
5974                    })
5975                }),
5976                replay: None,
5977                state_type: None,
5978            },
5979        );
5980    }
5981
5982    /// Register a workflow with one Serde request value and a Serde result.
5983    ///
5984    /// This is an ergonomic adapter over the same fixed Avro Value protocol as
5985    /// [`Worker::register_workflow_avro_value`]. It does not create or publish a
5986    /// workflow-specific schema. A task must contain zero arguments for a unit
5987    /// request or exactly one argument for every other request type.
5988    ///
5989    /// See the runnable
5990    /// [`hello_world` example](https://github.com/durable-workflow/sdk-rust/blob/main/examples/hello_world.rs)
5991    /// for typed workflow and activity contracts with retry and timeout policy.
5992    pub fn register_typed_workflow<I, O, F, Fut>(
5993        &mut self,
5994        workflow_type: impl Into<String>,
5995        handler: F,
5996    ) where
5997        I: DeserializeOwned + Send + 'static,
5998        O: Serialize + Send + 'static,
5999        F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6000        Fut: Future<Output = Result<O>> + Send + 'static,
6001    {
6002        let workflow_type = workflow_type.into();
6003        let handler_name = workflow_type.clone();
6004        let handler = Arc::new(handler);
6005        self.workflows.insert(
6006            workflow_type,
6007            RegisteredWorkflow {
6008                execute: Arc::new(move |ctx, input| {
6009                    let handler = Arc::clone(&handler);
6010                    let handler_name = handler_name.clone();
6011                    Box::pin(async move {
6012                        let input =
6013                            decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6014                        let result = handler(ctx, input).await?;
6015                        encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6016                    })
6017                }),
6018                replay: None,
6019                state_type: None,
6020            },
6021        );
6022    }
6023
6024    /// Register a workflow on the lossless fixed Avro Value surface.
6025    pub fn register_workflow_avro_value<F, Fut>(
6026        &mut self,
6027        workflow_type: impl Into<String>,
6028        handler: F,
6029    ) where
6030        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6031        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6032    {
6033        self.workflows.insert(
6034            workflow_type.into(),
6035            RegisteredWorkflow {
6036                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6037                replay: None,
6038                state_type: None,
6039            },
6040        );
6041    }
6042
6043    /// Register a workflow whose typed instance state can be reconstructed for queries.
6044    ///
6045    /// `state_factory` creates a fresh instance for every normal workflow task and
6046    /// query replay. The workflow handler is the single source of truth for state
6047    /// transitions: it updates [`WorkflowInstance`] after activities and signals
6048    /// resolve. Query replay runs this same handler over committed history and
6049    /// discards any commands it would emit.
6050    pub fn register_replayed_workflow<S, Factory, F, Fut>(
6051        &mut self,
6052        workflow_type: impl Into<String>,
6053        state_factory: Factory,
6054        handler: F,
6055    ) where
6056        S: Clone + Send + Sync + 'static,
6057        Factory: Fn() -> S + Send + Sync + 'static,
6058        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6059        Fut: Future<Output = Result<Value>> + Send + 'static,
6060    {
6061        let state_factory = Arc::new(state_factory);
6062        let handler = Arc::new(handler);
6063
6064        let execute_factory = Arc::clone(&state_factory);
6065        let execute_handler = Arc::clone(&handler);
6066        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6067            let state = WorkflowInstance::new(execute_factory());
6068            let handler = Arc::clone(&execute_handler);
6069            Box::pin(async move {
6070                let result = handler(ctx, input.into_json()?, state).await?;
6071                AvroValue::from_serialize(&result)
6072            }) as WorkflowFuture
6073        });
6074
6075        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6076            let state = WorkflowInstance::new(state_factory());
6077            let snapshot_state = state.clone();
6078            let snapshot: WorkflowStateSnapshot =
6079                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6080            let replay_handler = Arc::clone(&handler);
6081            let future = async move {
6082                let result = replay_handler(ctx, input.into_json()?, state).await?;
6083                AvroValue::from_serialize(&result)
6084            };
6085            ReplayedWorkflowInvocation {
6086                future: Box::pin(future),
6087                snapshot,
6088            }
6089        });
6090
6091        self.workflows.insert(
6092            workflow_type.into(),
6093            RegisteredWorkflow {
6094                execute,
6095                replay: Some(replay),
6096                state_type: Some(TypeId::of::<S>()),
6097            },
6098        );
6099    }
6100
6101    /// Register a replayable workflow with one Serde request value and result.
6102    ///
6103    /// Normal task execution and instance-state query replay both decode and
6104    /// encode through the fixed Avro Value codec. The state factory and handler
6105    /// otherwise follow [`Worker::register_replayed_workflow`].
6106    pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6107        &mut self,
6108        workflow_type: impl Into<String>,
6109        state_factory: Factory,
6110        handler: F,
6111    ) where
6112        I: DeserializeOwned + Send + 'static,
6113        O: Serialize + Send + 'static,
6114        S: Clone + Send + Sync + 'static,
6115        Factory: Fn() -> S + Send + Sync + 'static,
6116        F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6117        Fut: Future<Output = Result<O>> + Send + 'static,
6118    {
6119        let workflow_type = workflow_type.into();
6120        let state_factory = Arc::new(state_factory);
6121        let handler = Arc::new(handler);
6122
6123        let execute_name = workflow_type.clone();
6124        let execute_factory = Arc::clone(&state_factory);
6125        let execute_handler = Arc::clone(&handler);
6126        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6127            let state = WorkflowInstance::new(execute_factory());
6128            let handler = Arc::clone(&execute_handler);
6129            let handler_name = execute_name.clone();
6130            Box::pin(async move {
6131                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6132                let result = handler(ctx, input, state).await?;
6133                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6134            }) as WorkflowFuture
6135        });
6136
6137        let replay_name = workflow_type.clone();
6138        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6139            let state = WorkflowInstance::new(state_factory());
6140            let snapshot_state = state.clone();
6141            let snapshot: WorkflowStateSnapshot =
6142                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6143            let handler = Arc::clone(&handler);
6144            let handler_name = replay_name.clone();
6145            let future = async move {
6146                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6147                let result = handler(ctx, input, state).await?;
6148                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6149            };
6150            ReplayedWorkflowInvocation {
6151                future: Box::pin(future),
6152                snapshot,
6153            }
6154        });
6155
6156        self.workflows.insert(
6157            workflow_type,
6158            RegisteredWorkflow {
6159                execute,
6160                replay: Some(replay),
6161                state_type: Some(TypeId::of::<S>()),
6162            },
6163        );
6164    }
6165
6166    /// Register a replayable workflow on the lossless fixed Avro Value surface.
6167    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6168        &mut self,
6169        workflow_type: impl Into<String>,
6170        state_factory: Factory,
6171        handler: F,
6172    ) where
6173        S: Clone + Send + Sync + 'static,
6174        Factory: Fn() -> S + Send + Sync + 'static,
6175        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6176        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6177    {
6178        let state_factory = Arc::new(state_factory);
6179        let handler = Arc::new(handler);
6180
6181        let execute_factory = Arc::clone(&state_factory);
6182        let execute_handler = Arc::clone(&handler);
6183        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6184            let state = WorkflowInstance::new(execute_factory());
6185            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6186        });
6187
6188        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6189            let state = WorkflowInstance::new(state_factory());
6190            let snapshot_state = state.clone();
6191            let snapshot: WorkflowStateSnapshot =
6192                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6193            ReplayedWorkflowInvocation {
6194                future: Box::pin(handler(ctx, input, state)),
6195                snapshot,
6196            }
6197        });
6198
6199        self.workflows.insert(
6200            workflow_type.into(),
6201            RegisteredWorkflow {
6202                execute,
6203                replay: Some(replay),
6204                state_type: Some(TypeId::of::<S>()),
6205            },
6206        );
6207    }
6208
6209    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6210    where
6211        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6212        Fut: Future<Output = Result<Value>> + Send + 'static,
6213    {
6214        let handler = Arc::new(handler);
6215        self.activities.insert(
6216            activity_type.into(),
6217            Arc::new(move |ctx, args| {
6218                let handler = Arc::clone(&handler);
6219                Box::pin(async move {
6220                    let result = handler(ctx, args.into_json()?).await?;
6221                    AvroValue::from_serialize(&result)
6222                })
6223            }),
6224        );
6225    }
6226
6227    /// Register an activity with one Serde request value and a Serde result.
6228    ///
6229    /// Inputs and results use the platform's fixed Avro Value schema. Shape
6230    /// mismatches and unsupported Serde values return [`Error::HandlerType`]
6231    /// with the activity name and Rust type.
6232    pub fn register_typed_activity<I, O, F, Fut>(
6233        &mut self,
6234        activity_type: impl Into<String>,
6235        handler: F,
6236    ) where
6237        I: DeserializeOwned + Send + 'static,
6238        O: Serialize + Send + 'static,
6239        F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6240        Fut: Future<Output = Result<O>> + Send + 'static,
6241    {
6242        let activity_type = activity_type.into();
6243        let handler_name = activity_type.clone();
6244        let handler = Arc::new(handler);
6245        self.activities.insert(
6246            activity_type,
6247            Arc::new(move |ctx, input| {
6248                let handler = Arc::clone(&handler);
6249                let handler_name = handler_name.clone();
6250                Box::pin(async move {
6251                    let input =
6252                        decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6253                    let result = handler(ctx, input).await?;
6254                    encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6255                })
6256            }),
6257        );
6258    }
6259
6260    /// Register an activity on the lossless fixed Avro Value surface.
6261    pub fn register_activity_avro_value<F, Fut>(
6262        &mut self,
6263        activity_type: impl Into<String>,
6264        handler: F,
6265    ) where
6266        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6267        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6268    {
6269        self.activities.insert(
6270            activity_type.into(),
6271            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6272        );
6273    }
6274
6275    /// Register a named, read-only query handler for a workflow type.
6276    ///
6277    /// The workflow type must also be registered with [`Worker::register_workflow`]
6278    /// before the worker runs. The handler receives only an immutable committed
6279    /// state snapshot and normalized query arguments.
6280    pub fn register_query<F, Fut>(
6281        &mut self,
6282        workflow_type: impl Into<String>,
6283        query_name: impl Into<String>,
6284        handler: F,
6285    ) where
6286        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6287        Fut: Future<Output = Result<Value>> + Send + 'static,
6288    {
6289        let handler = Arc::new(handler);
6290        self.queries
6291            .entry(workflow_type.into())
6292            .or_default()
6293            .insert(
6294                query_name.into(),
6295                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6296                    let handler = Arc::clone(&handler);
6297                    Box::pin(async move {
6298                        let result = handler(ctx, args.into_json()?).await?;
6299                        AvroValue::from_serialize(&result)
6300                    })
6301                })),
6302            );
6303    }
6304
6305    /// Register a query handler on the lossless fixed Avro Value surface.
6306    pub fn register_query_avro_value<F, Fut>(
6307        &mut self,
6308        workflow_type: impl Into<String>,
6309        query_name: impl Into<String>,
6310        handler: F,
6311    ) where
6312        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6313        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6314    {
6315        self.queries
6316            .entry(workflow_type.into())
6317            .or_default()
6318            .insert(
6319                query_name.into(),
6320                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6321            );
6322    }
6323
6324    /// Register a named query against deterministically replayed instance state.
6325    ///
6326    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
6327    /// same state type `S`. The handler receives an immutable, detached state
6328    /// clone, so successful and failed queries cannot affect workflow execution
6329    /// or the state reconstructed by a later query.
6330    pub fn register_replayed_query<S, F, Fut>(
6331        &mut self,
6332        workflow_type: impl Into<String>,
6333        query_name: impl Into<String>,
6334        handler: F,
6335    ) where
6336        S: Clone + Send + Sync + 'static,
6337        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6338        Fut: Future<Output = Result<Value>> + Send + 'static,
6339    {
6340        let handler = Arc::new(handler);
6341        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6342            let state = state.downcast::<S>().map_err(|_| {
6343                "registered query state type does not match the replayed workflow state".to_string()
6344            })?;
6345            let handler = Arc::clone(&handler);
6346            Ok(Box::pin(async move {
6347                let result = handler(ctx, state, args.into_json()?).await?;
6348                AvroValue::from_serialize(&result)
6349            }))
6350        });
6351
6352        self.queries
6353            .entry(workflow_type.into())
6354            .or_default()
6355            .insert(
6356                query_name.into(),
6357                RegisteredQuery::Replayed {
6358                    state_type: TypeId::of::<S>(),
6359                    handler: erased_handler,
6360                },
6361            );
6362    }
6363
6364    /// Register a replayed-state query on the lossless fixed Avro Value surface.
6365    pub fn register_replayed_query_avro_value<S, F, Fut>(
6366        &mut self,
6367        workflow_type: impl Into<String>,
6368        query_name: impl Into<String>,
6369        handler: F,
6370    ) where
6371        S: Clone + Send + Sync + 'static,
6372        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6373        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6374    {
6375        let handler = Arc::new(handler);
6376        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6377            let state = state.downcast::<S>().map_err(|_| {
6378                "registered query state type does not match the replayed workflow state".to_string()
6379            })?;
6380            Ok(Box::pin(handler(ctx, state, args)))
6381        });
6382
6383        self.queries
6384            .entry(workflow_type.into())
6385            .or_default()
6386            .insert(
6387                query_name.into(),
6388                RegisteredQuery::Replayed {
6389                    state_type: TypeId::of::<S>(),
6390                    handler: erased_handler,
6391                },
6392            );
6393    }
6394
6395    /// Register a synchronous workflow update handler.
6396    pub fn register_update<F, Fut>(
6397        &mut self,
6398        workflow_type: impl Into<String>,
6399        update_name: impl Into<String>,
6400        handler: F,
6401    ) where
6402        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6403        Fut: Future<Output = Result<Value>> + Send + 'static,
6404    {
6405        let handler = Arc::new(handler);
6406        self.updates
6407            .entry(workflow_type.into())
6408            .or_default()
6409            .insert(
6410                update_name.into(),
6411                Arc::new(move |ctx, args| {
6412                    let handler = Arc::clone(&handler);
6413                    Box::pin(async move {
6414                        let result = handler(ctx, args.into_json()?).await?;
6415                        AvroValue::from_serialize(&result)
6416                    })
6417                }),
6418            );
6419    }
6420
6421    /// Register an update handler on the lossless fixed Avro Value surface.
6422    pub fn register_update_avro_value<F, Fut>(
6423        &mut self,
6424        workflow_type: impl Into<String>,
6425        update_name: impl Into<String>,
6426        handler: F,
6427    ) where
6428        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6429        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6430    {
6431        self.updates
6432            .entry(workflow_type.into())
6433            .or_default()
6434            .insert(
6435                update_name.into(),
6436                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6437            );
6438    }
6439
6440    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6441        let mut command_contracts = serde_json::Map::new();
6442        for workflow_type in self.workflows.keys() {
6443            let mut queries = self
6444                .queries
6445                .get(workflow_type)
6446                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6447                .unwrap_or_default();
6448            queries.sort();
6449            let mut updates = self
6450                .updates
6451                .get(workflow_type)
6452                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6453                .unwrap_or_default();
6454            updates.sort();
6455            command_contracts.insert(
6456                workflow_type.clone(),
6457                json!({
6458                    "queries": queries,
6459                    "query_contracts": [],
6460                    "signals": [],
6461                    "signal_contracts": [],
6462                    "updates": updates,
6463                    "update_contracts": [],
6464                    "update_validators": [],
6465                }),
6466            );
6467        }
6468
6469        self.client
6470            .register_worker_with_command_contracts(
6471                &self.worker_id,
6472                &self.task_queue,
6473                self.workflows.keys().cloned().collect(),
6474                self.activities.keys().cloned().collect(),
6475                self.max_concurrent_workflow_tasks,
6476                self.max_concurrent_activity_tasks,
6477                [
6478                    Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6479                    Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6480                    Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6481                    Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6482                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6483                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6484                    worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6485                        .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6486                ]
6487                .into_iter()
6488                .flatten()
6489                .collect(),
6490                Value::Object(command_contracts),
6491            )
6492            .await
6493    }
6494
6495    /// Run until shutdown or a terminal worker error occurs.
6496    ///
6497    /// Empty long-poll expirations do not stop the worker. Retryable poll and
6498    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
6499    /// authentication, protocol, and other non-retryable failures are returned.
6500    pub async fn run(&self) -> Result<()> {
6501        self.run_until(std::future::pending::<()>()).await
6502    }
6503
6504    /// Run until `shutdown` resolves or a terminal worker error occurs.
6505    ///
6506    /// This has the same liveness and terminal-error contract as [`Worker::run`].
6507    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6508    where
6509        F: Future<Output = ()>,
6510    {
6511        let stop = Arc::new(AtomicBool::new(false));
6512        let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6513        let worker = self.with_storage_admission(Arc::clone(&stop));
6514        let run = worker.run_with_storage_admission(Arc::clone(&stop));
6515        tokio::pin!(run);
6516        tokio::pin!(shutdown);
6517        tokio::select! {
6518            result = &mut run => result,
6519            _ = &mut shutdown => {
6520                stop.store(true, Ordering::SeqCst);
6521                run.await
6522            }
6523        }
6524    }
6525
6526    fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6527        let mut worker = self.clone();
6528        worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6529            policy: self.retry_policy,
6530            stop,
6531        });
6532        worker
6533    }
6534
6535    async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6536        let registration = self.register().await?;
6537        if !registration.registered {
6538            return Err(Error::WorkerLoop(format!(
6539                "worker registration for {:?} was not accepted",
6540                self.worker_id
6541            )));
6542        }
6543        let registered_worker_id = registration.worker_id.clone();
6544        let primary = self.run_registered_until(stop, registration).await;
6545        let deregistration = self
6546            .client
6547            .deregister_worker_registration(&registered_worker_id)
6548            .await;
6549
6550        match (primary, deregistration) {
6551            (Ok(()), Ok(_)) => Ok(()),
6552            (Ok(()), Err(deregistration)) => Err(deregistration),
6553            (Err(primary), Ok(_)) => Err(primary),
6554            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6555                primary: Box::new(primary),
6556                deregistration: Box::new(deregistration),
6557            }),
6558        }
6559    }
6560
6561    async fn run_registered_until(
6562        &self,
6563        stop: Arc<AtomicBool>,
6564        registration: RegisterWorkerResponse,
6565    ) -> Result<()> {
6566        let heartbeat_interval = Duration::from_secs(
6567            registration
6568                .heartbeat_interval_seconds
6569                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6570        );
6571        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
6572        // from the completion of the preceding attempt, including its bounded
6573        // retries. A fixed-epoch interval can leave an already-due tick queued
6574        // while an acknowledgement is slow, producing a catch-up heartbeat as
6575        // soon as that request completes.
6576        let heartbeat = tokio::time::sleep(Duration::ZERO);
6577        tokio::pin!(heartbeat);
6578        // Poll responses may already have leased server-side work by the time
6579        // they become ready, so each poller owns its responses through
6580        // completion or failure instead of racing raw polls in this select.
6581        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6582            let worker = self.clone();
6583            let stop = Arc::clone(&stop);
6584            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6585        });
6586        let mut activity_poller = (!self.activities.is_empty()).then(|| {
6587            let worker = self.clone();
6588            let stop = Arc::clone(&stop);
6589            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6590        });
6591        let mut query_poller = (!self.queries.is_empty()).then(|| {
6592            let worker = self.clone();
6593            let stop = Arc::clone(&stop);
6594            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6595        });
6596
6597        loop {
6598            tokio::select! {
6599                _ = wait_for_worker_stop(&stop) => {
6600                    stop.store(true, Ordering::SeqCst);
6601                    break;
6602                }
6603                _ = &mut heartbeat => {
6604                    let result = self.retry_worker_operation(|| {
6605                        self.client.heartbeat_worker(
6606                            &self.worker_id,
6607                            self.max_concurrent_workflow_tasks,
6608                            self.max_concurrent_activity_tasks,
6609                        )
6610                    }).await;
6611                    heartbeat
6612                        .as_mut()
6613                        .reset(tokio::time::Instant::now() + heartbeat_interval);
6614                    match result {
6615                        Ok(acknowledgement) => {
6616                            if let Some(observer) = &self.heartbeat_observer {
6617                                observer(&WorkerHeartbeatObservation {
6618                                    worker_id: self.worker_id.clone(),
6619                                    task_queue: self.task_queue.clone(),
6620                                    acknowledged_at_unix_millis: SystemTime::now()
6621                                        .duration_since(UNIX_EPOCH)
6622                                        .unwrap_or_default()
6623                                        .as_millis()
6624                                        .min(u64::MAX as u128)
6625                                        as u64,
6626                                    acknowledgement,
6627                                });
6628                            }
6629                        }
6630                        Err(error) => {
6631                            stop.store(true, Ordering::SeqCst);
6632                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6633                            return Err(error);
6634                        }
6635                    }
6636                }
6637                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6638                    workflow_poller = None;
6639                    let stopped_by_server = stop.load(Ordering::SeqCst);
6640                    stop.store(true, Ordering::SeqCst);
6641                    let poller_result = optional_poller_result("workflow", result);
6642                    let join_result =
6643                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6644                    poller_result?;
6645                    join_result?;
6646                    if stopped_by_server {
6647                        return Ok(());
6648                    }
6649                    return Err(Error::WorkerLoop(
6650                        "workflow poller stopped unexpectedly".to_string(),
6651                    ));
6652                }
6653                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6654                    activity_poller = None;
6655                    let stopped_by_server = stop.load(Ordering::SeqCst);
6656                    stop.store(true, Ordering::SeqCst);
6657                    let poller_result = optional_poller_result("activity", result);
6658                    let join_result =
6659                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6660                    poller_result?;
6661                    join_result?;
6662                    if stopped_by_server {
6663                        return Ok(());
6664                    }
6665                    return Err(Error::WorkerLoop(
6666                        "activity poller stopped unexpectedly".to_string(),
6667                    ));
6668                }
6669                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6670                    query_poller = None;
6671                    let stopped_by_server = stop.load(Ordering::SeqCst);
6672                    stop.store(true, Ordering::SeqCst);
6673                    let poller_result = optional_poller_result("query", result);
6674                    let join_result =
6675                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6676                    poller_result?;
6677                    join_result?;
6678                    if stopped_by_server {
6679                        return Ok(());
6680                    }
6681                    return Err(Error::WorkerLoop(
6682                        "query poller stopped unexpectedly".to_string(),
6683                    ));
6684                }
6685            }
6686        }
6687
6688        join_pollers(
6689            workflow_poller.take(),
6690            activity_poller.take(),
6691            query_poller.take(),
6692        )
6693        .await
6694    }
6695
6696    /// Poll and settle at most one task from each enabled task family.
6697    ///
6698    /// A workflow may reach its server-enforced run deadline while this worker
6699    /// holds a task. When the completion endpoint authoritatively rejects that
6700    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
6701    /// terminal `run_status=failed`, the workflow tick is considered settled:
6702    /// the late command was not recorded and cannot replace the terminal run.
6703    /// Every other completion rejection remains an error. This worker-level
6704    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
6705    /// only bounds how long a client waits for a result.
6706    ///
6707    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
6708    /// the original [`Error::Http`] status and response body.
6709    pub async fn run_once(&self) -> Result<usize> {
6710        let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
6711        let mut handled = 0;
6712        match worker.poll_workflow_once().await? {
6713            ManagedPollOutcome::Handled => handled += 1,
6714            ManagedPollOutcome::Stop => return Ok(handled),
6715            ManagedPollOutcome::Idle => {}
6716        }
6717        match worker.poll_activity_once().await? {
6718            ManagedPollOutcome::Handled => handled += 1,
6719            ManagedPollOutcome::Stop => return Ok(handled),
6720            ManagedPollOutcome::Idle => {}
6721        }
6722        if !self.queries.is_empty() {
6723            match worker.poll_query_once().await? {
6724                ManagedPollOutcome::Handled => handled += 1,
6725                ManagedPollOutcome::Stop => return Ok(handled),
6726                ManagedPollOutcome::Idle => {}
6727            }
6728        }
6729        Ok(handled)
6730    }
6731
6732    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6733        let poll_request_id = unique_request_id("rust-workflow-poll");
6734        let response = self
6735            .retry_worker_operation(|| {
6736                self.client.poll_workflow_task_response_with_request_id(
6737                    &self.worker_id,
6738                    &self.task_queue,
6739                    self.poll_timeout,
6740                    &poll_request_id,
6741                    0,
6742                )
6743            })
6744            .await;
6745        let Some(response) = self.settle_worker_poll_response(response).await? else {
6746            return Ok(ManagedPollOutcome::Idle);
6747        };
6748        if response.outcome().should_stop() {
6749            return Ok(ManagedPollOutcome::Stop);
6750        }
6751        let memo_updates_supported =
6752            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6753        let Some(task) = response.task else {
6754            return Ok(ManagedPollOutcome::Idle);
6755        };
6756
6757        let task_id = task.task_id.clone();
6758        let attempt = task.workflow_task_attempt;
6759        let run_id = task.run_id.clone();
6760        let lease_owner = task
6761            .lease_owner
6762            .clone()
6763            .unwrap_or_else(|| self.worker_id.clone());
6764
6765        match self.execute_workflow_task_decision(task) {
6766            Ok(decision)
6767                if commands_use_workflow_memo_updates(&decision.commands)
6768                    && !memo_updates_supported =>
6769            {
6770                self.client
6771                    .fail_workflow_task(
6772                        &task_id,
6773                        &lease_owner,
6774                        attempt,
6775                        Error::WorkflowMemoUpdatesUnavailable.to_string(),
6776                    )
6777                    .await?;
6778            }
6779            Ok(decision) if decision.commands.is_empty() => {
6780                // A replay can consume a recorded pending durable command
6781                // without producing a new command. The standalone protocol
6782                // acknowledges that state through the typed waiting outcome;
6783                // an empty completion is rejected by servers that require at
6784                // least one executable command.
6785                self.client
6786                    .fail_workflow_task_with_type(
6787                        &task_id,
6788                        &lease_owner,
6789                        attempt,
6790                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6791                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6792                    )
6793                    .await?;
6794            }
6795            Ok(decision) => {
6796                let completion = self
6797                    .client
6798                    .complete_workflow_task_with_message_streams(
6799                        &task_id,
6800                        &lease_owner,
6801                        attempt,
6802                        decision.commands,
6803                        decision.message_stream_cursors,
6804                        decision.message_stream_waits,
6805                    )
6806                    .await;
6807                if let Err(error) = completion {
6808                    if !workflow_task_completion_is_terminal_timeout(
6809                        &error,
6810                        &task_id,
6811                        attempt,
6812                        run_id.as_deref(),
6813                    ) {
6814                        return Err(error);
6815                    }
6816                }
6817            }
6818            Err(error) => {
6819                self.client
6820                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6821                    .await?;
6822            }
6823        }
6824
6825        Ok(ManagedPollOutcome::Handled)
6826    }
6827
6828    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6829        while !stop.load(Ordering::SeqCst) {
6830            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6831                stop.store(true, Ordering::SeqCst);
6832                break;
6833            }
6834        }
6835
6836        Ok(())
6837    }
6838
6839    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6840        let poll_request_id = unique_request_id("rust-activity-poll");
6841        let response = self
6842            .retry_worker_operation(|| {
6843                self.client.poll_activity_task_response_with_request_id(
6844                    &self.worker_id,
6845                    &self.task_queue,
6846                    self.poll_timeout,
6847                    &poll_request_id,
6848                    0,
6849                )
6850            })
6851            .await;
6852        let Some(response) = self.settle_worker_poll_response(response).await? else {
6853            return Ok(ManagedPollOutcome::Idle);
6854        };
6855        if response.outcome().should_stop() {
6856            return Ok(ManagedPollOutcome::Stop);
6857        }
6858        let Some(task) = response.task else {
6859            return Ok(ManagedPollOutcome::Idle);
6860        };
6861
6862        let task_id = task.task_id.clone();
6863        let attempt_id = task
6864            .activity_attempt_id
6865            .clone()
6866            .or(task.attempt_id.clone())
6867            .unwrap_or_default();
6868        let lease_owner = task
6869            .lease_owner
6870            .clone()
6871            .unwrap_or_else(|| self.worker_id.clone());
6872        let codec = task.payload_codec.clone();
6873        let result = self.execute_activity_task(task).await;
6874        match result {
6875            Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
6876            Ok(value) => {
6877                let completion = self
6878                    .client
6879                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6880                    .await;
6881                if let Err(error) = completion {
6882                    if !activity_task_rejection_is_final(&error) {
6883                        return Err(error);
6884                    }
6885                }
6886            }
6887            Err(error) => {
6888                let failure = self
6889                    .client
6890                    .fail_activity_task(
6891                        &task_id,
6892                        &attempt_id,
6893                        &lease_owner,
6894                        error.to_string(),
6895                        false,
6896                    )
6897                    .await;
6898                if let Err(error) = failure {
6899                    if !activity_task_rejection_is_final(&error) {
6900                        return Err(error);
6901                    }
6902                }
6903            }
6904        }
6905
6906        Ok(ManagedPollOutcome::Handled)
6907    }
6908
6909    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6910        while !stop.load(Ordering::SeqCst) {
6911            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6912                stop.store(true, Ordering::SeqCst);
6913                break;
6914            }
6915        }
6916
6917        Ok(())
6918    }
6919
6920    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6921        let poll_request_id = unique_request_id("rust-query-poll");
6922        let response = self
6923            .retry_worker_operation(|| {
6924                self.client.poll_query_task_response_with_request_id(
6925                    &self.worker_id,
6926                    &self.task_queue,
6927                    self.poll_timeout,
6928                    &poll_request_id,
6929                    0,
6930                )
6931            })
6932            .await;
6933        let Some(response) = self.settle_worker_poll_response(response).await? else {
6934            return Ok(ManagedPollOutcome::Idle);
6935        };
6936        if response.outcome().should_stop() {
6937            return Ok(ManagedPollOutcome::Stop);
6938        }
6939        let Some(task) = response.task else {
6940            return Ok(ManagedPollOutcome::Idle);
6941        };
6942
6943        let query_task_id = task.query_task_id.clone();
6944        let attempt = task.query_task_attempt;
6945        let lease_owner = task
6946            .lease_owner
6947            .clone()
6948            .unwrap_or_else(|| self.worker_id.clone());
6949        let codec = task.payload_codec.clone();
6950
6951        match self.execute_query_task(task).await {
6952            Ok(value) => {
6953                let result_envelope = match encode_typed_envelope(&value, &codec) {
6954                    Ok(result_envelope) => result_envelope,
6955                    Err(error) => {
6956                        let failure = self
6957                            .client
6958                            .fail_query_task(
6959                                &query_task_id,
6960                                &lease_owner,
6961                                attempt,
6962                                error.to_string(),
6963                                "query_result_encode_failed",
6964                                "QueryResultEncodeFailed",
6965                            )
6966                            .await;
6967                        if let Err(error) = failure {
6968                            if !query_task_rejection_is_final(&error) {
6969                                return Err(error);
6970                            }
6971                        }
6972                        return Ok(ManagedPollOutcome::Handled);
6973                    }
6974                };
6975
6976                if let Err(error) = self
6977                    .client
6978                    .complete_query_task_with_envelope(
6979                        &query_task_id,
6980                        &lease_owner,
6981                        attempt,
6982                        value.clone().into_json()?,
6983                        result_envelope,
6984                    )
6985                    .await
6986                {
6987                    if !query_task_rejection_is_final(&error) {
6988                        return Err(error);
6989                    }
6990                }
6991            }
6992            Err(failure) => {
6993                let result = self
6994                    .client
6995                    .fail_query_task(
6996                        &query_task_id,
6997                        &lease_owner,
6998                        attempt,
6999                        failure.message,
7000                        failure.reason,
7001                        failure.failure_type,
7002                    )
7003                    .await;
7004                if let Err(error) = result {
7005                    if !query_task_rejection_is_final(&error) {
7006                        return Err(error);
7007                    }
7008                }
7009            }
7010        }
7011
7012        Ok(ManagedPollOutcome::Handled)
7013    }
7014
7015    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7016        while !stop.load(Ordering::SeqCst) {
7017            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
7018                stop.store(true, Ordering::SeqCst);
7019                break;
7020            }
7021        }
7022
7023        Ok(())
7024    }
7025
7026    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7027    where
7028        F: FnMut() -> Fut,
7029        Fut: Future<Output = Result<T>>,
7030    {
7031        let mut retries = 0;
7032
7033        loop {
7034            match operation().await {
7035                Err(error)
7036                    if worker_operation_is_retryable(&error)
7037                        && retries < self.retry_policy.max_retries =>
7038                {
7039                    retries += 1;
7040                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7041                }
7042                result => return result,
7043            }
7044        }
7045    }
7046
7047    async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7048        match response {
7049            Ok(response) => Ok(Some(response)),
7050            Err(error) => {
7051                let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7052                    return Err(error);
7053                };
7054                let minimum_delay = self
7055                    .retry_policy
7056                    .initial_backoff
7057                    .max(Duration::from_millis(1));
7058                let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7059                tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7060                Ok(None)
7061            }
7062        }
7063    }
7064
7065    async fn execute_query_task(
7066        &self,
7067        mut task: QueryTask,
7068    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7069        validate_query_task_payloads(&task).map_err(|error| {
7070            QueryTaskExecutionFailure::new(
7071                "query_payload_decode_failed",
7072                error.to_string(),
7073                "QueryPayloadDecodeFailed",
7074            )
7075        })?;
7076
7077        if !self.workflows.contains_key(&task.workflow_type) {
7078            return Err(QueryTaskExecutionFailure::new(
7079                "query_workflow_type_not_registered",
7080                format!("no workflow registered for type {:?}", task.workflow_type),
7081                "WorkflowTypeNotRegistered",
7082            ));
7083        }
7084
7085        let Some(handlers) = self.queries.get(&task.workflow_type) else {
7086            return Err(QueryTaskExecutionFailure::new(
7087                "query_handler_unavailable",
7088                format!(
7089                    "query handlers are unavailable for workflow type {:?}",
7090                    task.workflow_type
7091                ),
7092                "QueryHandlerUnavailable",
7093            ));
7094        };
7095        let Some(query) = handlers.get(&task.query_name) else {
7096            return Err(QueryTaskExecutionFailure::new(
7097                "rejected_unknown_query",
7098                format!("unknown query {:?}", task.query_name),
7099                "QueryFailed",
7100            ));
7101        };
7102
7103        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7104            .map_err(|error| {
7105            QueryTaskExecutionFailure::new(
7106                "query_payload_decode_failed",
7107                format!("cannot decode query arguments: {error}"),
7108                "QueryPayloadDecodeFailed",
7109            )
7110        })?;
7111        let workflow_input_typed =
7112            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7113                .map_err(|error| {
7114                    QueryTaskExecutionFailure::new(
7115                        "query_workflow_state_unavailable",
7116                        format!("cannot decode workflow start input: {error}"),
7117                        "QueryWorkflowStateUnavailable",
7118                    )
7119                })?;
7120        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7121            QueryTaskExecutionFailure::new(
7122                "query_workflow_state_unavailable",
7123                format!("cannot project workflow start input: {error}"),
7124                "QueryWorkflowStateUnavailable",
7125            )
7126        })?;
7127        hydrate_query_history_from_export(&mut task).map_err(|error| {
7128            QueryTaskExecutionFailure::new(
7129                "query_workflow_state_unavailable",
7130                format!("cannot restore query history snapshot: {error}"),
7131                "QueryWorkflowStateUnavailable",
7132            )
7133        })?;
7134        enrich_query_history_from_export(&mut task).map_err(|error| {
7135            QueryTaskExecutionFailure::new(
7136                "query_workflow_state_unavailable",
7137                format!("cannot restore compact query history payloads: {error}"),
7138                "QueryWorkflowStateUnavailable",
7139            )
7140        })?;
7141        let signal_events = query_signal_events(&task).map_err(|error| {
7142            QueryTaskExecutionFailure::new(
7143                "query_workflow_state_unavailable",
7144                format!("cannot decode committed workflow signals: {error}"),
7145                "QueryWorkflowStateUnavailable",
7146            )
7147        })?;
7148        let history_events = Arc::new(std::mem::take(&mut task.history_events));
7149        let context = QueryContext {
7150            workflow_id: task.workflow_id,
7151            run_id: task.run_id,
7152            workflow_type: task.workflow_type.clone(),
7153            run_status: task.run_status,
7154            workflow_input,
7155            workflow_input_avro_value: workflow_input_typed.clone(),
7156            history_events: Arc::clone(&history_events),
7157            signal_events: Arc::new(signal_events),
7158        };
7159
7160        let future = match query {
7161            RegisteredQuery::Snapshot(handler) => handler(context, args),
7162            RegisteredQuery::Replayed {
7163                state_type,
7164                handler,
7165            } => {
7166                let workflow = self
7167                    .workflows
7168                    .get(&task.workflow_type)
7169                    .expect("workflow registration was checked above");
7170                if workflow.state_type != Some(*state_type) {
7171                    return Err(QueryTaskExecutionFailure::new(
7172                        "query_workflow_state_unavailable",
7173                        "replayed query state type does not match its workflow registration",
7174                        "QueryWorkflowStateUnavailable",
7175                    ));
7176                }
7177                let replay = workflow.replay.as_ref().ok_or_else(|| {
7178                    QueryTaskExecutionFailure::new(
7179                        "query_workflow_state_unavailable",
7180                        format!(
7181                            "workflow type {:?} is not registered for instance-state replay",
7182                            task.workflow_type
7183                        ),
7184                        "QueryWorkflowStateUnavailable",
7185                    )
7186                })?;
7187                let workflow_state = Arc::new(Mutex::new(
7188                    WorkflowState::new_with_identity(
7189                        history_events.as_ref().clone(),
7190                        context.workflow_id.clone(),
7191                        context.run_id.clone(),
7192                        self.task_queue.clone(),
7193                        task.payload_codec,
7194                        None,
7195                    )
7196                    .map_err(|error| {
7197                        QueryTaskExecutionFailure::new(
7198                            "query_workflow_state_unavailable",
7199                            format!("workflow replay failed before query: {error}"),
7200                            "QueryWorkflowStateUnavailable",
7201                        )
7202                    })?,
7203                ));
7204                let workflow_context = WorkflowContext {
7205                    state: workflow_state,
7206                };
7207                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7208                let mut cx = TaskContext::from_waker(noop_waker_ref());
7209                match invocation.future.as_mut().poll(&mut cx) {
7210                    Poll::Ready(Ok(_)) => {
7211                        workflow_context
7212                            .ensure_history_consumed()
7213                            .map_err(|error| {
7214                                QueryTaskExecutionFailure::new(
7215                                    "query_workflow_state_unavailable",
7216                                    format!("workflow replay failed before query: {error}"),
7217                                    "QueryWorkflowStateUnavailable",
7218                                )
7219                            })?;
7220                    }
7221                    Poll::Ready(Err(error)) => {
7222                        return Err(QueryTaskExecutionFailure::new(
7223                            "query_workflow_state_unavailable",
7224                            format!("workflow replay failed before query: {error}"),
7225                            "QueryWorkflowStateUnavailable",
7226                        ));
7227                    }
7228                    Poll::Pending => {
7229                        let commands = workflow_context.take_commands().map_err(|error| {
7230                            QueryTaskExecutionFailure::new(
7231                                "query_workflow_state_unavailable",
7232                                format!("workflow replay failed before query: {error}"),
7233                                "QueryWorkflowStateUnavailable",
7234                            )
7235                        })?;
7236                        if commands.is_empty()
7237                            && !workflow_context
7238                                .matched_recorded_pending()
7239                                .map_err(|error| {
7240                                    QueryTaskExecutionFailure::new(
7241                                        "query_workflow_state_unavailable",
7242                                        format!("workflow replay failed before query: {error}"),
7243                                        "QueryWorkflowStateUnavailable",
7244                                    )
7245                                })?
7246                        {
7247                            return Err(QueryTaskExecutionFailure::new(
7248                                "query_workflow_state_unavailable",
7249                                "workflow replay yielded without a durable command",
7250                                "QueryWorkflowStateUnavailable",
7251                            ));
7252                        }
7253                    }
7254                }
7255                let state = (invocation.snapshot)().map_err(|error| {
7256                    QueryTaskExecutionFailure::new(
7257                        "query_workflow_state_unavailable",
7258                        format!("cannot snapshot replayed workflow state: {error}"),
7259                        "QueryWorkflowStateUnavailable",
7260                    )
7261                })?;
7262                handler(context, state, args).map_err(|message| {
7263                    QueryTaskExecutionFailure::new(
7264                        "query_workflow_state_unavailable",
7265                        message,
7266                        "QueryWorkflowStateUnavailable",
7267                    )
7268                })?
7269            }
7270        };
7271
7272        future.await.map_err(|error| {
7273            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7274        })
7275    }
7276
7277    #[cfg(test)]
7278    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7279        Ok(self.execute_workflow_task_decision(task)?.commands)
7280    }
7281
7282    fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7283        validate_workflow_task_payloads(&task)?;
7284
7285        if let Some(update_id) = task
7286            .workflow_update_id
7287            .as_deref()
7288            .filter(|update_id| !update_id.is_empty())
7289        {
7290            return self
7291                .execute_update_task(&task, update_id)
7292                .map(WorkflowTaskDecision::without_message_streams);
7293        }
7294
7295        let workflow = self
7296            .workflows
7297            .get(&task.workflow_type)
7298            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7299        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7300        let resume_signal = decode_resume_signal(&task)?;
7301        let history_budget = WorkflowHistoryBudget {
7302            event_count: task
7303                .total_history_events
7304                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7305            size_bytes: task.history_size_bytes,
7306            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7307            pressure: task.history_budget_pressure.clone(),
7308        };
7309        let workflow_command_identity = task
7310            .workflow_command_id
7311            .clone()
7312            .filter(|identity| !identity.is_empty())
7313            .unwrap_or_default();
7314        let mut workflow_state = WorkflowState::new_with_identity(
7315            task.history_events,
7316            task.workflow_id,
7317            task.run_id,
7318            self.task_queue.clone(),
7319            task.payload_codec.clone(),
7320            resume_signal,
7321        )?;
7322        workflow_state.history_budget = history_budget;
7323        workflow_state.workflow_command_identity = workflow_command_identity;
7324        workflow_state.cancel_requested = task.cancel_requested;
7325        let state = Arc::new(Mutex::new(workflow_state));
7326        let ctx = WorkflowContext { state };
7327        let mut future = (workflow.execute)(ctx.clone(), input);
7328        let mut cx = TaskContext::from_waker(noop_waker_ref());
7329
7330        match future.as_mut().poll(&mut cx) {
7331            Poll::Ready(Ok(result)) => {
7332                ctx.ensure_history_consumed()?;
7333                let result = encode_typed_envelope(&result, &task.payload_codec)?;
7334                let mut commands = ctx.take_commands()?;
7335                commands.push(json!({
7336                    "type": "complete_workflow",
7337                    "result": result
7338                }));
7339                self.message_stream_decision(&ctx, commands)
7340            }
7341            Poll::Ready(Err(error)) => {
7342                if let Error::ContinueAsNew(request) = error {
7343                    let mut commands = ctx.take_commands()?;
7344                    if let Some(command) = ctx.continue_as_new_command(request)? {
7345                        commands.push(command);
7346                    }
7347                    ctx.ensure_history_consumed()?;
7348                    return self.message_stream_decision(&ctx, commands);
7349                }
7350                if workflow_task_integrity_error(&error) {
7351                    // Replay and protocol failures describe the workflow-task
7352                    // decision itself. Preserve their specific failure reason
7353                    // instead of replacing it with the derivative fact that
7354                    // recorded commands remain unconsumed.
7355                    return Err(error);
7356                }
7357                // A handler error must not hide a committed durable command that
7358                // upgraded workflow code no longer consumes.
7359                ctx.ensure_history_consumed()?;
7360                let mut commands = ctx.take_commands()?;
7361                commands.push(workflow_failure_command(&error));
7362                self.message_stream_decision(&ctx, commands)
7363            }
7364            Poll::Pending => {
7365                let commands = ctx.take_commands()?;
7366                if commands.is_empty() && !ctx.matched_recorded_pending()? {
7367                    Err(Error::WorkflowYieldedWithoutCommand)
7368                } else {
7369                    self.message_stream_decision(&ctx, commands)
7370                }
7371            }
7372        }
7373    }
7374
7375    fn message_stream_decision(
7376        &self,
7377        ctx: &WorkflowContext,
7378        commands: Vec<Value>,
7379    ) -> Result<WorkflowTaskDecision> {
7380        let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7381        Ok(WorkflowTaskDecision {
7382            commands,
7383            message_stream_cursors,
7384            message_stream_waits,
7385        })
7386    }
7387
7388    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7389        if !self.workflows.contains_key(&task.workflow_type) {
7390            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7391        }
7392
7393        let accepted = task.history_events.iter().rev().find_map(|event| {
7394            (event.event_type == "UpdateAccepted"
7395                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7396            .then_some(&event.payload)
7397        });
7398        let update_name = accepted
7399            .and_then(|payload| payload.get("update_name"))
7400            .and_then(Value::as_str)
7401            .or(task.update_name.as_deref())
7402            .unwrap_or_default();
7403        let Some(handler) = self
7404            .updates
7405            .get(&task.workflow_type)
7406            .and_then(|handlers| handlers.get(update_name))
7407        else {
7408            return Ok(vec![json!({
7409                "type": "fail_update",
7410                "update_id": update_id,
7411                "message": format!(
7412                    "no update handler is registered for {}.{update_name}",
7413                    task.workflow_type
7414                ),
7415                "exception_type": "UnknownUpdate",
7416                "non_retryable": true,
7417            })]);
7418        };
7419        let arguments = accepted
7420            .and_then(|payload| payload.get("arguments"))
7421            .or(task.arguments.as_ref());
7422        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7423        let context = QueryContext {
7424            workflow_id: task.workflow_id.clone(),
7425            run_id: task.run_id.clone(),
7426            workflow_type: task.workflow_type.clone(),
7427            run_status: Some("running".to_string()),
7428            workflow_input: Value::Null,
7429            workflow_input_avro_value: AvroValue::Null,
7430            history_events: Arc::new(task.history_events.clone()),
7431            signal_events: Arc::new(Vec::new()),
7432        };
7433        let mut future = handler(context, arguments);
7434        let mut cx = TaskContext::from_waker(noop_waker_ref());
7435
7436        match future.as_mut().poll(&mut cx) {
7437            Poll::Ready(Ok(result)) => Ok(vec![json!({
7438                "type": "complete_update",
7439                "update_id": update_id,
7440                "result": encode_typed_envelope(&result, &task.payload_codec)?,
7441            })]),
7442            Poll::Ready(Err(error)) => Ok(vec![json!({
7443                "type": "fail_update",
7444                "update_id": update_id,
7445                "message": error.to_string(),
7446                "exception_type": "UpdateFailed",
7447                "non_retryable": true,
7448            })]),
7449            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7450        }
7451    }
7452
7453    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7454        validate_activity_task_payloads(&task)?;
7455
7456        let handler = self
7457            .activities
7458            .get(&task.activity_type)
7459            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7460        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7461        let attempt_id = task
7462            .activity_attempt_id
7463            .clone()
7464            .or(task.attempt_id.clone())
7465            .unwrap_or_default();
7466        let lease_owner = task
7467            .lease_owner
7468            .clone()
7469            .unwrap_or_else(|| self.worker_id.clone());
7470        let ctx = ActivityContext {
7471            client: self.client.clone(),
7472            task_id: task.task_id,
7473            activity_attempt_id: attempt_id,
7474            lease_owner,
7475            activity_type: task.activity_type,
7476            attempt_number: task.attempt_number,
7477            task_queue: self.task_queue.clone(),
7478            worker_id: self.worker_id.clone(),
7479        };
7480
7481        handler(ctx, args).await
7482    }
7483}
7484
7485fn poller_result(
7486    kind: &str,
7487    result: std::result::Result<Result<()>, tokio::task::JoinError>,
7488) -> Result<()> {
7489    match result {
7490        Ok(result) => result,
7491        Err(error) => Err(Error::WorkerLoop(format!(
7492            "{kind} poller join error: {error}"
7493        ))),
7494    }
7495}
7496
7497fn optional_poller_result(
7498    kind: &str,
7499    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7500) -> Result<()> {
7501    match result {
7502        Some(result) => poller_result(kind, result),
7503        None => Ok(()),
7504    }
7505}
7506
7507async fn join_pollers(
7508    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7509    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7510    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7511) -> Result<()> {
7512    let mut first_error = None;
7513
7514    if let Some(handle) = workflow_poller {
7515        if let Err(error) = poller_result("workflow", handle.await) {
7516            first_error.get_or_insert(error);
7517        }
7518    }
7519
7520    if let Some(handle) = activity_poller {
7521        if let Err(error) = poller_result("activity", handle.await) {
7522            first_error.get_or_insert(error);
7523        }
7524    }
7525
7526    if let Some(handle) = query_poller {
7527        if let Err(error) = poller_result("query", handle.await) {
7528            first_error.get_or_insert(error);
7529        }
7530    }
7531
7532    if let Some(error) = first_error {
7533        Err(error)
7534    } else {
7535        Ok(())
7536    }
7537}
7538
7539fn default_worker_id() -> String {
7540    let millis = SystemTime::now()
7541        .duration_since(UNIX_EPOCH)
7542        .unwrap_or_default()
7543        .as_millis();
7544    format!("rust-worker-{}-{millis}", std::process::id())
7545}
7546
7547fn percent_encode_path_segment(segment: &str) -> String {
7548    const HEX: &[u8; 16] = b"0123456789ABCDEF";
7549    let mut encoded = String::with_capacity(segment.len());
7550
7551    for byte in segment.bytes() {
7552        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7553            encoded.push(char::from(byte));
7554        } else {
7555            encoded.push('%');
7556            encoded.push(char::from(HEX[(byte >> 4) as usize]));
7557            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7558        }
7559    }
7560
7561    encoded
7562}
7563
7564fn unique_request_id(prefix: &str) -> String {
7565    let nanos = SystemTime::now()
7566        .duration_since(UNIX_EPOCH)
7567        .unwrap_or_default()
7568        .as_nanos();
7569    format!("{prefix}-{}-{nanos}", std::process::id())
7570}
7571
7572#[derive(Debug)]
7573struct QueryTaskExecutionFailure {
7574    reason: String,
7575    message: String,
7576    failure_type: String,
7577}
7578
7579impl QueryTaskExecutionFailure {
7580    fn new(
7581        reason: impl Into<String>,
7582        message: impl Into<String>,
7583        failure_type: impl Into<String>,
7584    ) -> Self {
7585        Self {
7586            reason: reason.into(),
7587            message: message.into(),
7588            failure_type: failure_type.into(),
7589        }
7590    }
7591}
7592
7593/// Typed local state owned by one deterministic workflow invocation.
7594///
7595/// Use [`WorkflowInstance::update`] for the same state transitions during
7596/// ordinary execution and replay. A replayed query receives a detached
7597/// immutable `Arc<S>` rather than this mutation-capable handle.
7598#[derive(Clone, Debug)]
7599pub struct WorkflowInstance<S> {
7600    state: Arc<Mutex<S>>,
7601}
7602
7603impl<S> WorkflowInstance<S> {
7604    fn new(state: S) -> Self {
7605        Self {
7606            state: Arc::new(Mutex::new(state)),
7607        }
7608    }
7609
7610    /// Read the current workflow-instance state without changing it.
7611    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7612        let state = self
7613            .state
7614            .lock()
7615            .map_err(|_| Error::WorkflowStatePoisoned)?;
7616        Ok(reader(&state))
7617    }
7618
7619    /// Apply one deterministic workflow-instance state transition.
7620    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7621        let mut state = self
7622            .state
7623            .lock()
7624            .map_err(|_| Error::WorkflowStatePoisoned)?;
7625        Ok(transition(&mut state))
7626    }
7627}
7628
7629impl<S: Clone> WorkflowInstance<S> {
7630    fn snapshot(&self) -> Result<S> {
7631        self.read(Clone::clone)
7632    }
7633}
7634
7635#[derive(Clone, Debug, PartialEq)]
7636pub struct MessageStreamMessage {
7637    pub stream_name: String,
7638    pub message_id: String,
7639    pub position: u64,
7640    pub arguments: Vec<AvroValue>,
7641}
7642
7643#[derive(Clone, Debug)]
7644pub struct MessageStream {
7645    ctx: WorkflowContext,
7646    name: String,
7647}
7648
7649impl MessageStream {
7650    /// Wait for one message, then return a bounded currently-available batch.
7651    pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7652        if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7653            return Err(Error::Codec(format!(
7654                "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7655            )));
7656        }
7657        loop {
7658            if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7659                return Ok(batch);
7660            }
7661
7662            self.ctx.record_message_stream_wait(&self.name)?;
7663            let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7664            let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7665            self.ctx.buffer_message_stream_delivery(arguments)?;
7666            if let Some(sequence) = replay_wait_sequence {
7667                self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7668            }
7669        }
7670    }
7671
7672    pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7673        self.receive(1)
7674            .await?
7675            .into_iter()
7676            .next()
7677            .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7678    }
7679}
7680
7681#[derive(Clone, Debug)]
7682pub struct WorkflowContext {
7683    state: Arc<Mutex<WorkflowState>>,
7684}
7685
7686fn valid_memo_key(key: &str) -> bool {
7687    let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7688
7689    !key.is_empty()
7690        && key.len() <= 64
7691        && (numeric_candidate.is_empty()
7692            || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7693        && key
7694            .bytes()
7695            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7696}
7697
7698fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7699    BASE64
7700        .decode(encode_avro_value(value)?.blob)
7701        .map(|bytes| bytes.len())
7702        .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7703}
7704
7705fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7706    let AvroValue::Map(entries) = value else {
7707        return Err(Error::InvalidMemoUpdate(
7708            "entries must serialize to an Avro string-keyed map".to_string(),
7709        ));
7710    };
7711    if require_entries && entries.is_empty() {
7712        return Err(Error::InvalidMemoUpdate(
7713            "at least one entry is required".to_string(),
7714        ));
7715    }
7716    if entries.len() > MAX_MEMO_ENTRIES {
7717        return Err(Error::InvalidMemoUpdate(format!(
7718            "at most {MAX_MEMO_ENTRIES} entries are allowed"
7719        )));
7720    }
7721
7722    for (key, value) in &entries {
7723        if !valid_memo_key(&key) {
7724            return Err(Error::InvalidMemoUpdate(
7725                "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7726            ));
7727        }
7728        if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7729            return Err(Error::InvalidMemoUpdate(format!(
7730                "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7731            )));
7732        }
7733    }
7734
7735    let value = AvroValue::Map(entries);
7736    if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7737        return Err(Error::InvalidMemoUpdate(format!(
7738            "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7739        )));
7740    }
7741    Ok(value)
7742}
7743
7744fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7745    let object = envelope.as_object().ok_or_else(|| {
7746        Error::InvalidMemoUpdate(
7747            "history field must use the public {codec, blob} payload envelope".to_string(),
7748        )
7749    })?;
7750    if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7751        return Err(Error::InvalidMemoUpdate(
7752            "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7753        ));
7754    }
7755
7756    canonical_memo_entries(
7757        decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7758        require_entries,
7759    )
7760}
7761
7762impl WorkflowContext {
7763    pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7764        let name = name.into();
7765        if name.is_empty()
7766            || name.len() > 128
7767            || !name.bytes().all(|byte| {
7768                byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7769            })
7770        {
7771            return Err(Error::Codec(
7772                "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7773                    .to_string(),
7774            ));
7775        }
7776        Ok(MessageStream {
7777            ctx: self.clone(),
7778            name,
7779        })
7780    }
7781
7782    fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7783        let mut state = self
7784            .state
7785            .lock()
7786            .map_err(|_| Error::WorkflowStatePoisoned)?;
7787        let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7788        state
7789            .message_stream_waits
7790            .insert(name.to_string(), position);
7791        Ok(())
7792    }
7793
7794    fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7795        let mut state = self
7796            .state
7797            .lock()
7798            .map_err(|_| Error::WorkflowStatePoisoned)?;
7799        let cursor = state
7800            .message_stream_cursors
7801            .get(&message.stream_name)
7802            .copied()
7803            .unwrap_or(0);
7804        if message.position <= cursor {
7805            return Ok(());
7806        }
7807        let pending = state
7808            .message_stream_messages
7809            .entry(message.stream_name.clone())
7810            .or_default();
7811        if pending.iter().any(|candidate| {
7812            candidate.position == message.position || candidate.message_id == message.message_id
7813        }) {
7814            return Ok(());
7815        }
7816        pending.push(message);
7817        pending.sort_by_key(|candidate| candidate.position);
7818        Ok(())
7819    }
7820
7821    fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7822        if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7823            match delivery {
7824                MessageStreamDelivery::Message(message) => {
7825                    let stream_name = message.stream_name.clone();
7826                    self.buffer_message_stream(message)?;
7827                    return Ok(Some(stream_name));
7828                }
7829                MessageStreamDelivery::Cursor {
7830                    stream_name,
7831                    through_position,
7832                } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7833            }
7834        }
7835        Ok(None)
7836    }
7837
7838    fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7839        let state = self
7840            .state
7841            .lock()
7842            .map_err(|_| Error::WorkflowStatePoisoned)?;
7843        Ok(match state.recorded_commands.get(state.command_cursor) {
7844            Some(RecordedCommand::SignalWait {
7845                sequence,
7846                signal_name,
7847                ..
7848            }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7849            _ => None,
7850        })
7851    }
7852
7853    fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7854        let (history, payload_codec) = {
7855            let state = self
7856                .state
7857                .lock()
7858                .map_err(|_| Error::WorkflowStatePoisoned)?;
7859            (
7860                Arc::clone(&state.history_events),
7861                state.payload_codec.clone(),
7862            )
7863        };
7864
7865        let Some(opened_index) = history.iter().position(|event| {
7866            event.event_type == "SignalWaitOpened"
7867                && durable_event_sequence(event) == Some(wait_sequence)
7868                && event.payload.get("signal_name").and_then(Value::as_str)
7869                    == Some(MESSAGE_STREAM_SIGNAL)
7870        }) else {
7871            return Ok(());
7872        };
7873        let boundary_index = history
7874            .iter()
7875            .enumerate()
7876            .skip(opened_index + 1)
7877            .find_map(|(index, event)| {
7878                (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7879                    && is_authored_command_open_event(event))
7880                .then_some(index)
7881            })
7882            .unwrap_or(history.len());
7883
7884        for event in history[opened_index + 1..boundary_index]
7885            .iter()
7886            .filter(|event| {
7887                event.event_type == "SignalReceived"
7888                    && event.payload.get("signal_name").and_then(Value::as_str)
7889                        == Some(MESSAGE_STREAM_SIGNAL)
7890            })
7891        {
7892            let arguments = decode_signal_event_arguments(event, &payload_codec)?
7893                .into_iter()
7894                .map(AvroValue::into_json)
7895                .collect::<Result<Vec<_>>>()?;
7896            self.buffer_message_stream_delivery(arguments)?;
7897        }
7898        Ok(())
7899    }
7900
7901    fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7902        let mut state = self
7903            .state
7904            .lock()
7905            .map_err(|_| Error::WorkflowStatePoisoned)?;
7906        let cursor = state
7907            .message_stream_cursors
7908            .entry(name.to_string())
7909            .or_default();
7910        *cursor = (*cursor).max(through_position);
7911        if let Some(pending) = state.message_stream_messages.get_mut(name) {
7912            pending.retain(|message| message.position > through_position);
7913        }
7914        Ok(())
7915    }
7916
7917    fn take_message_stream_batch(
7918        &self,
7919        name: &str,
7920        max_items: usize,
7921    ) -> Result<Option<Vec<MessageStreamMessage>>> {
7922        let mut state = self
7923            .state
7924            .lock()
7925            .map_err(|_| Error::WorkflowStatePoisoned)?;
7926        let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7927        let pending = state
7928            .message_stream_messages
7929            .entry(name.to_string())
7930            .or_default();
7931        let count = contiguous_message_stream_count(pending, cursor, max_items);
7932        if count == 0 {
7933            return Ok(None);
7934        }
7935        let batch = pending.drain(..count).collect::<Vec<_>>();
7936        let position = batch.last().map(|message| message.position).unwrap_or(0);
7937        state
7938            .message_stream_cursors
7939            .insert(name.to_string(), position);
7940        state.message_stream_waits.remove(name);
7941        Ok(Some(batch))
7942    }
7943
7944    fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7945        let state = self
7946            .state
7947            .lock()
7948            .map_err(|_| Error::WorkflowStatePoisoned)?;
7949        let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7950        cursors.sort_by_key(|(name, _)| *name);
7951        let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7952        waits.sort_by_key(|(name, _)| *name);
7953        Ok((
7954            cursors
7955                .into_iter()
7956                .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7957                .collect(),
7958            waits
7959                .into_iter()
7960                .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7961                .collect(),
7962        ))
7963    }
7964    /// Identity of the parent workflow currently being replayed.
7965    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7966        let state = self
7967            .state
7968            .lock()
7969            .map_err(|_| Error::WorkflowStatePoisoned)?;
7970        Ok(WorkflowIdentity {
7971            workflow_id: state.workflow_id.clone(),
7972            run_id: state.run_id.clone(),
7973        })
7974    }
7975
7976    /// Return the server-published history budget for this workflow task.
7977    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7978        let state = self
7979            .state
7980            .lock()
7981            .map_err(|_| Error::WorkflowStatePoisoned)?;
7982        Ok(state.history_budget.clone())
7983    }
7984
7985    /// Continue this workflow instance as a fresh run with replacement arguments.
7986    ///
7987    /// Return this value directly from the workflow handler. The worker converts
7988    /// it to the terminal protocol command only after replay has consumed every
7989    /// recorded durable command.
7990    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7991        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7992    }
7993
7994    /// Continue as new with optional workflow-type and task-queue overrides.
7995    pub fn continue_as_new_with_options<T: Serialize>(
7996        &self,
7997        options: ContinueAsNewOptions,
7998        args: T,
7999    ) -> Result<Value> {
8000        options.validate()?;
8001        Err(Error::ContinueAsNew(ContinueAsNewRequest {
8002            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
8003            options,
8004        }))
8005    }
8006
8007    pub fn activity<T: Serialize>(
8008        &self,
8009        activity_type: impl Into<String>,
8010        args: T,
8011    ) -> ActivityCall {
8012        self.activity_with_options(activity_type, ActivityOptions::new(), args)
8013    }
8014
8015    pub fn activity_on_queue<T, Q>(
8016        &self,
8017        activity_type: impl Into<String>,
8018        task_queue: Option<Q>,
8019        args: T,
8020    ) -> ActivityCall
8021    where
8022        T: Serialize,
8023        Q: Into<String>,
8024    {
8025        let mut options = ActivityOptions::new();
8026        options.task_queue = task_queue.map(Into::into);
8027        self.activity_with_options(activity_type, options, args)
8028    }
8029
8030    /// Schedule one durable activity with retry, routing, and timeout options.
8031    ///
8032    /// Options are validated before the command is emitted. Once the command is
8033    /// recorded, replay consumes the same activity lifecycle at this command
8034    /// position and never emits a duplicate schedule.
8035    ///
8036    /// ```no_run
8037    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
8038    /// # use std::time::Duration;
8039    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
8040    /// let result = ctx
8041    ///     .activity_with_options(
8042    ///         "charge-card",
8043    ///         ActivityOptions::new()
8044    ///             .task_queue("payments")
8045    ///             .retry_policy(
8046    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
8047    ///                     Duration::from_secs(1),
8048    ///                     2,
8049    ///                     Some(Duration::from_secs(30)),
8050    ///                 ),
8051    ///             )
8052    ///             .start_to_close_timeout(Duration::from_secs(60))
8053    ///             .schedule_to_close_timeout(Duration::from_secs(180))
8054    ///             .heartbeat_timeout(Duration::from_secs(15)),
8055    ///         json!([{"order_id": "order-42"}]),
8056    ///     )
8057    ///     .await;
8058    /// match result {
8059    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
8060    ///         "reason": failure.reason,
8061    ///         "timeout_kind": failure.timeout_kind,
8062    ///     })),
8063    ///     other => other,
8064    /// }
8065    /// # }
8066    /// ```
8067    pub fn activity_with_options<T: Serialize>(
8068        &self,
8069        activity_type: impl Into<String>,
8070        options: ActivityOptions,
8071        args: T,
8072    ) -> ActivityCall {
8073        ActivityCall {
8074            ctx: self.clone(),
8075            activity_type: activity_type.into(),
8076            options,
8077            args: Some(AvroValue::from_serialize(&args)),
8078            scheduled: false,
8079            parallel_group_path: Vec::new(),
8080        }
8081    }
8082
8083    pub async fn activity_avro_value<T: Serialize>(
8084        &self,
8085        activity_type: impl Into<String>,
8086        args: T,
8087    ) -> Result<AvroValue> {
8088        let mut call = self.activity(activity_type, args);
8089        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8090    }
8091
8092    pub async fn activity_avro_value_with_options<T: Serialize>(
8093        &self,
8094        activity_type: impl Into<String>,
8095        options: ActivityOptions,
8096        args: T,
8097    ) -> Result<AvroValue> {
8098        let mut call = self.activity_with_options(activity_type, options, args);
8099        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8100    }
8101
8102    /// Schedule an activity with a Serde request and decode its Serde result.
8103    pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8104    where
8105        I: Serialize,
8106        O: DeserializeOwned,
8107    {
8108        self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8109            .await
8110    }
8111
8112    /// Schedule an activity with options and decode its result into `O`.
8113    ///
8114    /// Both directions use the fixed Avro Value codec. In particular, this
8115    /// method does not deserialize the JSON-safe inspection projection returned
8116    /// by the dynamic [`ActivityCall`] future.
8117    pub async fn activity_typed_with_options<I, O>(
8118        &self,
8119        activity_type: impl Into<String>,
8120        options: ActivityOptions,
8121        args: I,
8122    ) -> Result<O>
8123    where
8124        I: Serialize,
8125        O: DeserializeOwned,
8126    {
8127        let activity_type = activity_type.into();
8128        let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8129            handler_type_error::<I>(
8130                HandlerKind::Activity,
8131                &activity_type,
8132                HandlerValueKind::Input,
8133                error.to_string(),
8134            )
8135        });
8136        let mut call = ActivityCall {
8137            ctx: self.clone(),
8138            activity_type: activity_type.clone(),
8139            options,
8140            args: Some(encoded),
8141            scheduled: false,
8142            parallel_group_path: Vec::new(),
8143        };
8144        let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8145        decode_handler_result(result, HandlerKind::Activity, &activity_type)
8146    }
8147
8148    /// Schedule and join a deterministic activity/child/timer group.
8149    ///
8150    /// Nested groups retain their input shape. Every durable leaf is scheduled
8151    /// before this future yields, results are assembled by declaration order,
8152    /// and a failure returns [`Error::ParallelFailed`] with typed cause,
8153    /// declaration path, stable group metadata, and completed siblings.
8154    pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8155        ParallelCall::new(self.clone(), operations)
8156    }
8157
8158    /// Alias for [`WorkflowContext::parallel`].
8159    pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8160        self.parallel(operations)
8161    }
8162
8163    /// Lossless fixed-Avro variant of [`WorkflowContext::parallel`].
8164    pub async fn parallel_avro_value(
8165        &self,
8166        operations: Vec<ParallelOperation>,
8167    ) -> Result<Vec<ParallelAvroResult>> {
8168        let mut call = self.parallel(operations);
8169        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8170    }
8171
8172    /// Start every durable operation and resume from the one winner persisted
8173    /// by Server. Non-winning operations continue and remain addressable.
8174    pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8175        let operations = operations
8176            .into_iter()
8177            .enumerate()
8178            .map(|(index, operation)| (SelectionKey::Index(index), operation))
8179            .collect();
8180        SelectCall::new(self.clone(), operations)
8181    }
8182
8183    /// Named-key variant of [`WorkflowContext::select`].
8184    pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8185    where
8186        K: Into<SelectionKey>,
8187    {
8188        SelectCall::new(
8189            self.clone(),
8190            operations
8191                .into_iter()
8192                .map(|(key, operation)| (key.into(), operation))
8193                .collect(),
8194        )
8195    }
8196
8197    /// Create a workflow-local deterministic compensation registry.
8198    pub fn saga(&self) -> Saga {
8199        Saga::new(self.clone())
8200    }
8201
8202    /// Whether the current workflow task carries a cooperative cancel request.
8203    pub fn is_cancellation_requested(&self) -> Result<bool> {
8204        let state = self
8205            .state
8206            .lock()
8207            .map_err(|_| Error::WorkflowStatePoisoned)?;
8208        Ok(state.cancel_requested)
8209    }
8210
8211    /// Raise a typed cooperative cancellation at an author-controlled point.
8212    ///
8213    /// Passing this result to [`Saga::finish`] compensates already registered
8214    /// forward steps before the cancellation remains the initiating outcome.
8215    pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8216        if self.is_cancellation_requested()? {
8217            return Err(Error::WorkflowCancellationRequested(
8218                WorkflowCancellationRequested,
8219            ));
8220        }
8221        Ok(())
8222    }
8223
8224    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8225        SignalCall {
8226            ctx: self.clone(),
8227            signal_name: signal_name.into(),
8228            runtime_reserved_allowed: false,
8229            opened_wait: false,
8230            matched_pending: false,
8231            parallel_group_path: Vec::new(),
8232        }
8233    }
8234
8235    fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8236        SignalCall {
8237            ctx: self.clone(),
8238            signal_name: signal_name.into(),
8239            runtime_reserved_allowed: true,
8240            opened_wait: false,
8241            matched_pending: false,
8242            parallel_group_path: Vec::new(),
8243        }
8244    }
8245
8246    pub async fn wait_signal_avro_value(
8247        &self,
8248        signal_name: impl Into<String>,
8249    ) -> Result<Vec<AvroValue>> {
8250        let mut call = self.wait_signal(signal_name);
8251        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8252    }
8253
8254    /// Return every committed signal argument list with the given name.
8255    ///
8256    /// This history-backed view is deterministic and is intended for
8257    /// condition predicates that must be re-evaluated after a signal while the
8258    /// workflow is blocked on [`WorkflowContext::wait_condition`].
8259    pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8260        self.signals_avro_value(signal_name)?
8261            .into_iter()
8262            .map(|arguments| {
8263                arguments
8264                    .into_iter()
8265                    .map(AvroValue::into_json)
8266                    .collect::<Result<Vec<_>>>()
8267            })
8268            .collect()
8269    }
8270
8271    /// Lossless fixed Avro Value view of committed signals with the given name.
8272    pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8273        let state = self
8274            .state
8275            .lock()
8276            .map_err(|_| Error::WorkflowStatePoisoned)?;
8277        state
8278            .history_events
8279            .iter()
8280            .filter(|event| {
8281                event.event_type == "SignalReceived"
8282                    && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8283            })
8284            .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8285            .collect()
8286    }
8287
8288    /// Return every committed update argument list with the given name.
8289    ///
8290    /// Accepted and applied records for the same update ID are de-duplicated.
8291    /// A Server task created after an update therefore replays the workflow and
8292    /// re-evaluates an open condition without application polling.
8293    pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8294        self.updates_avro_value(update_name)?
8295            .into_iter()
8296            .map(|arguments| {
8297                arguments
8298                    .into_iter()
8299                    .map(AvroValue::into_json)
8300                    .collect::<Result<Vec<_>>>()
8301            })
8302            .collect()
8303    }
8304
8305    /// Lossless fixed Avro Value view of committed updates with the given name.
8306    pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8307        let state = self
8308            .state
8309            .lock()
8310            .map_err(|_| Error::WorkflowStatePoisoned)?;
8311        let mut seen = Vec::new();
8312        let mut updates = Vec::new();
8313        for event in state.history_events.iter() {
8314            if !matches!(
8315                event.event_type.as_str(),
8316                "UpdateAccepted" | "UpdateApplied"
8317            ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8318                || event.payload.get("arguments").is_none()
8319            {
8320                continue;
8321            }
8322            if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8323                if seen.iter().any(|recorded| recorded == update_id) {
8324                    continue;
8325                }
8326                seen.push(update_id.to_string());
8327            }
8328            updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8329        }
8330        Ok(updates)
8331    }
8332
8333    /// Wait for a deterministic predicate to become true or for its durable
8334    /// timeout to elapse.
8335    ///
8336    /// Prefer [`wait_condition!`] for inline predicates so changes to the Rust
8337    /// predicate tokens automatically change the recorded definition
8338    /// fingerprint. Direct callers must provide an equally stable identity in
8339    /// [`ConditionWaitOptions`].
8340    pub fn wait_condition<F>(
8341        &self,
8342        options: ConditionWaitOptions,
8343        predicate: F,
8344    ) -> ConditionWaitCall
8345    where
8346        F: Fn() -> Result<bool> + Send + 'static,
8347    {
8348        ConditionWaitCall {
8349            ctx: self.clone(),
8350            options,
8351            predicate: Box::new(predicate),
8352            occurrence_id: None,
8353            opened_wait: false,
8354            parallel_group_path: Vec::new(),
8355        }
8356    }
8357
8358    /// Wait for server-backed durable time without blocking the worker executor.
8359    ///
8360    /// Polling this future emits one `start_timer` command and yields. The
8361    /// server records the deadline, so neither worker nor server restarts reset
8362    /// the wait. Replay resolves the future only from a `TimerScheduled` and
8363    /// `TimerFired` pair at the same position in the shared durable-command
8364    /// stream, with matching sequence, timer identity, and delay. Sub-second
8365    /// durations round up because protocol deadlines use whole seconds.
8366    ///
8367    /// ```no_run
8368    /// # use durable_workflow::{json, Client, Worker};
8369    /// # use std::time::Duration;
8370    /// # fn configure(client: Client) {
8371    /// let mut worker = Worker::new(client, "rust-workers");
8372    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
8373    ///     ctx.sleep(Duration::from_secs(5)).await?;
8374    ///     Ok(json!({"status": "timer fired"}))
8375    /// });
8376    /// # }
8377    /// ```
8378    pub fn sleep(&self, duration: Duration) -> TimerCall {
8379        let delay_seconds = duration
8380            .as_secs()
8381            .checked_add(u64::from(duration.subsec_nanos() > 0));
8382        TimerCall {
8383            ctx: self.clone(),
8384            delay_seconds,
8385            scheduled: false,
8386            matched_pending: false,
8387            parallel_group_path: Vec::new(),
8388        }
8389    }
8390
8391    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
8392    pub fn start_timer(&self, duration: Duration) -> TimerCall {
8393        self.sleep(duration)
8394    }
8395
8396    /// Evaluate a non-deterministic callback once and durably record its typed value.
8397    ///
8398    /// On replay the callback is not invoked: the value is decoded from the
8399    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
8400    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
8401    /// small values that must remain fixed for the lifetime of a workflow run.
8402    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8403    where
8404        T: Serialize + DeserializeOwned,
8405        F: FnOnce() -> T,
8406    {
8407        {
8408            let mut state = self
8409                .state
8410                .lock()
8411                .map_err(|_| Error::WorkflowStatePoisoned)?;
8412            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8413                return match recorded {
8414                    RecordedCommand::SideEffect { sequence, value } => {
8415                        state.command_cursor += 1;
8416                        value.deserialize().map_err(|error| {
8417                            Error::NonDeterministicReplay(ReplayFailure::new(
8418                                "side_effect_type_mismatch",
8419                                Some(sequence),
8420                                Some(std::any::type_name::<T>().to_string()),
8421                                Some(error.to_string()),
8422                                "recorded side-effect value is incompatible with the requested Rust type",
8423                            ))
8424                        })
8425                    }
8426                    other => Err(command_mismatch(&other, "side effect")),
8427                };
8428            }
8429        }
8430
8431        let value = callback();
8432        let avro_value = AvroValue::from_serialize(&value)?;
8433        let mut state = self
8434            .state
8435            .lock()
8436            .map_err(|_| Error::WorkflowStatePoisoned)?;
8437        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8438        state.commands.push(json!({
8439            "type": "record_side_effect",
8440            "result": result,
8441        }));
8442        Ok(value)
8443    }
8444
8445    /// Record or replay a lossless fixed Avro Value side effect.
8446    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8447    where
8448        F: FnOnce() -> AvroValue,
8449    {
8450        {
8451            let mut state = self
8452                .state
8453                .lock()
8454                .map_err(|_| Error::WorkflowStatePoisoned)?;
8455            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8456                return match recorded {
8457                    RecordedCommand::SideEffect { value, .. } => {
8458                        state.command_cursor += 1;
8459                        Ok(value)
8460                    }
8461                    other => Err(command_mismatch(&other, "side effect")),
8462                };
8463            }
8464        }
8465
8466        let value = callback();
8467        let mut state = self
8468            .state
8469            .lock()
8470            .map_err(|_| Error::WorkflowStatePoisoned)?;
8471        let result = encode_typed_envelope(&value, &state.payload_codec)?;
8472        state.commands.push(json!({
8473            "type": "record_side_effect",
8474            "result": result,
8475        }));
8476        Ok(value)
8477    }
8478
8479    /// Append output items at a deterministic workflow command boundary.
8480    ///
8481    /// Stable idempotency keys are derived from the server-provided durable
8482    /// workflow command identity, command ordinal, and item index. Replay
8483    /// consumes the recorded side effect and never emits another append.
8484    pub fn append_workflow_stream(
8485        &self,
8486        stream_name: impl Into<String>,
8487        items: &[WorkflowStreamAppendItem],
8488        max_pending_items: Option<u64>,
8489    ) -> Result<()> {
8490        if items.is_empty() {
8491            return Err(Error::Codec(
8492                "workflow_stream_items_empty: append requires at least one item".to_string(),
8493            ));
8494        }
8495        if max_pending_items == Some(0) {
8496            return Err(Error::Codec(
8497                "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8498                    .to_string(),
8499            ));
8500        }
8501        let stream_name = stream_name.into();
8502        if stream_name.is_empty() {
8503            return Err(Error::Codec(
8504                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8505            ));
8506        }
8507
8508        let mut state = self
8509            .state
8510            .lock()
8511            .map_err(|_| Error::WorkflowStatePoisoned)?;
8512        let command_ordinal = state.workflow_stream_command_counter;
8513        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8514            state.workflow_stream_command_counter += 1;
8515            return match recorded {
8516                RecordedCommand::SideEffect { .. } => {
8517                    state.command_cursor += 1;
8518                    Ok(())
8519                }
8520                other => Err(command_mismatch(&other, "workflow stream append")),
8521            };
8522        }
8523
8524        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8525        state.workflow_stream_command_counter += 1;
8526        let wire_items = items
8527            .iter()
8528            .enumerate()
8529            .map(|(item_index, item)| {
8530                item.wire_value(Some(format!(
8531                    "dw-stream:{identity}:{command_ordinal}:{item_index}"
8532                )))
8533            })
8534            .collect::<Vec<_>>();
8535        let mut directive = json!({
8536            "operation": "append",
8537            "stream_name": stream_name,
8538            "command_identity": identity,
8539            "command_ordinal": command_ordinal,
8540            "items": wire_items,
8541        });
8542        if let Some(max_pending_items) = max_pending_items {
8543            directive["max_pending_items"] = json!(max_pending_items);
8544        }
8545        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8546        state.commands.push(json!({
8547            "type": "record_side_effect",
8548            "result": result,
8549            "workflow_stream": directive,
8550        }));
8551        Ok(())
8552    }
8553
8554    /// Close a run-scoped output stream at a deterministic command boundary.
8555    pub fn close_workflow_stream(
8556        &self,
8557        stream_name: impl Into<String>,
8558        retention_seconds: Option<u64>,
8559    ) -> Result<()> {
8560        self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8561    }
8562
8563    /// Mark a run-scoped output stream errored at a deterministic command boundary.
8564    pub fn error_workflow_stream(
8565        &self,
8566        stream_name: impl Into<String>,
8567        error_reason: impl Into<String>,
8568        retention_seconds: Option<u64>,
8569    ) -> Result<()> {
8570        let error_reason = error_reason.into();
8571        if error_reason.is_empty() {
8572            return Err(Error::Codec(
8573                "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8574            ));
8575        }
8576        self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8577    }
8578
8579    fn finish_workflow_stream(
8580        &self,
8581        stream_name: String,
8582        error_reason: Option<String>,
8583        retention_seconds: Option<u64>,
8584    ) -> Result<()> {
8585        if stream_name.is_empty() {
8586            return Err(Error::Codec(
8587                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8588            ));
8589        }
8590        if retention_seconds == Some(0) {
8591            return Err(Error::Codec(
8592                "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8593            ));
8594        }
8595        let mut state = self
8596            .state
8597            .lock()
8598            .map_err(|_| Error::WorkflowStatePoisoned)?;
8599        let command_ordinal = state.workflow_stream_command_counter;
8600        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8601            state.workflow_stream_command_counter += 1;
8602            return match recorded {
8603                RecordedCommand::SideEffect { .. } => {
8604                    state.command_cursor += 1;
8605                    Ok(())
8606                }
8607                other => Err(command_mismatch(&other, "workflow stream close")),
8608            };
8609        }
8610        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8611        state.workflow_stream_command_counter += 1;
8612        let mut directive = json!({
8613            "operation": if error_reason.is_some() { "error" } else { "close" },
8614            "stream_name": stream_name,
8615            "command_identity": identity,
8616            "command_ordinal": command_ordinal,
8617        });
8618        if let Some(error_reason) = error_reason {
8619            directive["error_reason"] = json!(error_reason);
8620        }
8621        if let Some(retention_seconds) = retention_seconds {
8622            directive["retention_seconds"] = json!(retention_seconds);
8623        }
8624        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8625        state.commands.push(json!({
8626            "type": "record_side_effect",
8627            "result": result,
8628            "workflow_stream": directive,
8629        }));
8630        Ok(())
8631    }
8632
8633    fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8634        let identity = state.workflow_command_identity.as_str();
8635        if identity.is_empty() {
8636            return Err(Error::MissingWorkflowCommandIdentity);
8637        }
8638        Ok(identity)
8639    }
8640
8641    /// Validate, emit, or replay a typed workflow search-attribute update.
8642    ///
8643    /// The command is non-blocking within a workflow decision, but its
8644    /// `SearchAttributesUpserted` event occupies the same deterministic command
8645    /// stream as activities, timers, conditions, and other durable operations.
8646    pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8647        update.validate()?;
8648        let (attributes, attribute_types) = update.into_wire_parts();
8649        let mut state = self
8650            .state
8651            .lock()
8652            .map_err(|_| Error::WorkflowStatePoisoned)?;
8653
8654        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8655            return match recorded {
8656                RecordedCommand::SearchAttributes {
8657                    sequence,
8658                    attributes: recorded_attributes,
8659                    attribute_types: recorded_attribute_types,
8660                } => {
8661                    if recorded_attributes != attributes {
8662                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8663                            "search_attribute_value_mismatch",
8664                            Some(sequence),
8665                            Some(recorded_attributes.to_string()),
8666                            Some(attributes.to_string()),
8667                            "search-attribute values differ from the recorded durable command",
8668                        )));
8669                    }
8670                    if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8671                        if recorded_types != attribute_types {
8672                            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8673                                "search_attribute_type_mismatch",
8674                                Some(sequence),
8675                                Some(json!(recorded_types).to_string()),
8676                                Some(json!(attribute_types).to_string()),
8677                                "search-attribute declared types differ from the recorded durable command",
8678                            )));
8679                        }
8680                    }
8681                    state.command_cursor += 1;
8682                    Ok(())
8683                }
8684                other => Err(command_mismatch(&other, "search-attribute update")),
8685            };
8686        }
8687
8688        let mut command = serde_json::Map::from_iter([
8689            ("type".to_string(), json!("upsert_search_attributes")),
8690            ("attributes".to_string(), attributes),
8691        ]);
8692        if !attribute_types.is_empty() {
8693            command.insert("attribute_types".to_string(), json!(attribute_types));
8694        }
8695        state.commands.push(Value::Object(command));
8696        Ok(())
8697    }
8698
8699    /// Record a UUIDv4 once and return the same UUID on every replay.
8700    pub fn uuid_v4(&self) -> Result<Uuid> {
8701        self.side_effect(Uuid::new_v4)
8702    }
8703
8704    /// Select the newest supported version for a change, or replay the version
8705    /// already committed for that stable change ID.
8706    pub fn get_version(
8707        &self,
8708        change_id: impl Into<String>,
8709        min_supported: i32,
8710        max_supported: i32,
8711    ) -> Result<i32> {
8712        let change_id = change_id.into();
8713        if change_id.trim().is_empty() {
8714            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8715                "version_change_id_invalid",
8716                None,
8717                Some("non-empty change ID".to_string()),
8718                Some(change_id),
8719                "version markers require a stable non-empty change ID",
8720            )));
8721        }
8722        if min_supported > max_supported {
8723            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8724                "version_range_invalid",
8725                None,
8726                Some("min_supported <= max_supported".to_string()),
8727                Some(format!("{min_supported}..={max_supported}")),
8728                "version marker supported range is invalid",
8729            )));
8730        }
8731
8732        let mut state = self
8733            .state
8734            .lock()
8735            .map_err(|_| Error::WorkflowStatePoisoned)?;
8736        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8737            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8738            return Ok(version);
8739        }
8740
8741        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8742            return match recorded {
8743                RecordedCommand::VersionMarker {
8744                    sequence,
8745                    change_id: recorded_change_id,
8746                    version,
8747                    ..
8748                } => {
8749                    if recorded_change_id != change_id {
8750                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8751                            "version_change_id_mismatch",
8752                            Some(sequence),
8753                            Some(recorded_change_id),
8754                            Some(change_id),
8755                            "recorded version marker change ID differs from current workflow code",
8756                        )));
8757                    }
8758                    ensure_version_supported(
8759                        &change_id,
8760                        version,
8761                        min_supported,
8762                        max_supported,
8763                        sequence,
8764                    )?;
8765                    state.command_cursor += 1;
8766                    state.version_markers.insert(change_id, (version, sequence));
8767                    Ok(version)
8768                }
8769                other => Err(command_mismatch(
8770                    &other,
8771                    format!("version marker:{change_id}"),
8772                )),
8773            };
8774        }
8775
8776        let version = max_supported;
8777        state.commands.push(json!({
8778            "type": "record_version_marker",
8779            "change_id": change_id,
8780            "version": version,
8781            "min_supported": min_supported,
8782            "max_supported": max_supported,
8783        }));
8784        // Sequence numbers are assigned by the server. Zero identifies a marker
8785        // selected in this uncommitted decision batch for duplicate-call checks.
8786        state.version_markers.insert(change_id, (version, 0));
8787        Ok(version)
8788    }
8789
8790    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
8791    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8792        Ok(self.get_version(change_id, -1, 1)? == 1)
8793    }
8794
8795    /// Keep a patch marker in history after the legacy branch has been removed.
8796    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8797        self.get_version(change_id, -1, 1).map(|_| ())
8798    }
8799
8800    /// Merge non-indexed workflow memo metadata through durable history.
8801    ///
8802    /// Avro `null` deletes a key. The SDK encodes the complete patch in the
8803    /// public Avro payload envelope consumed by Server and Cloud runtimes.
8804    pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8805        let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8806        let mut state = self
8807            .state
8808            .lock()
8809            .map_err(|_| Error::WorkflowStatePoisoned)?;
8810
8811        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8812            return match recorded {
8813                RecordedCommand::Memo {
8814                    sequence,
8815                    entries: recorded_entries,
8816                } => {
8817                    if recorded_entries != entries {
8818                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8819                            "memo_update_mismatch",
8820                            Some(sequence),
8821                            Some(format!("{recorded_entries:?}")),
8822                            Some(format!("{entries:?}")),
8823                            "recorded memo entries differ from the current workflow update",
8824                        )));
8825                    }
8826                    state.command_cursor += 1;
8827                    Ok(())
8828                }
8829                other => Err(command_mismatch(&other, "memo upsert")),
8830            };
8831        }
8832
8833        let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8834        state.commands.push(json!({
8835            "type": "upsert_memo",
8836            "entries": entries_envelope,
8837        }));
8838        Ok(())
8839    }
8840
8841    /// Start a named durable child on an explicit queue and await its result.
8842    ///
8843    /// The command is recorded in the parent's sequence-ordered durable command
8844    /// stream. Replay keeps a scheduled child pending without emitting another
8845    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
8846    /// values preserve the history payload codec and include both sides of the
8847    /// durable relationship; failures are returned as
8848    /// [`Error::ChildWorkflowFailed`].
8849    ///
8850    /// ```no_run
8851    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
8852    /// # fn configure(client: Client) {
8853    /// let mut worker = Worker::new(client, "parent-workers");
8854    /// worker.register_workflow("order-parent", |ctx, _input| async move {
8855    ///     let child = ctx
8856    ///         .start_child_workflow(
8857    ///             "fulfil-order",
8858    ///             ChildWorkflowOptions::new("fulfilment-workers")
8859    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
8860    ///             json!([{"order_id": "order-42"}]),
8861    ///         )
8862    ///         .await?;
8863    ///     Ok(child.result)
8864    /// });
8865    /// # }
8866    /// ```
8867    pub fn start_child_workflow<T: Serialize>(
8868        &self,
8869        workflow_type: impl Into<String>,
8870        options: ChildWorkflowOptions,
8871        args: T,
8872    ) -> ChildWorkflowCall {
8873        ChildWorkflowCall {
8874            ctx: self.clone(),
8875            workflow_type: workflow_type.into(),
8876            options,
8877            args: Some(AvroValue::from_serialize(&args)),
8878            scheduled: false,
8879            matched_pending: false,
8880            parallel_group_path: Vec::new(),
8881        }
8882    }
8883
8884    pub async fn start_child_workflow_avro_value<T: Serialize>(
8885        &self,
8886        workflow_type: impl Into<String>,
8887        options: ChildWorkflowOptions,
8888        args: T,
8889    ) -> Result<ChildWorkflowAvroResult> {
8890        let mut call = self.start_child_workflow(workflow_type, options, args);
8891        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8892    }
8893
8894    fn take_commands(&self) -> Result<Vec<Value>> {
8895        let mut state = self
8896            .state
8897            .lock()
8898            .map_err(|_| Error::WorkflowStatePoisoned)?;
8899        Ok(std::mem::take(&mut state.commands))
8900    }
8901
8902    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8903        let mut state = self
8904            .state
8905            .lock()
8906            .map_err(|_| Error::WorkflowStatePoisoned)?;
8907
8908        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8909            return Err(command_mismatch(&recorded, "continue as new"));
8910        }
8911        if state.recorded_continue_as_new_sequence.is_some() {
8912            state.continue_as_new_consumed = true;
8913            return Ok(None);
8914        }
8915
8916        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8917        let mut command = serde_json::Map::from_iter([
8918            ("type".to_string(), json!("continue_as_new")),
8919            ("arguments".to_string(), arguments),
8920            ("queue".to_string(), json!(state.task_queue.clone())),
8921        ]);
8922        if let Some(workflow_type) = request.options.workflow_type {
8923            command.insert("workflow_type".to_string(), json!(workflow_type));
8924        }
8925        if let Some(task_queue) = request.options.task_queue {
8926            command.insert("queue".to_string(), json!(task_queue));
8927        }
8928        Ok(Some(Value::Object(command)))
8929    }
8930
8931    fn matched_recorded_pending(&self) -> Result<bool> {
8932        let state = self
8933            .state
8934            .lock()
8935            .map_err(|_| Error::WorkflowStatePoisoned)?;
8936        Ok(state.matched_recorded_pending)
8937    }
8938
8939    fn ensure_history_consumed(&self) -> Result<()> {
8940        let state = self
8941            .state
8942            .lock()
8943            .map_err(|_| Error::WorkflowStatePoisoned)?;
8944        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8945            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8946                "recorded_commands_unconsumed",
8947                Some(command.sequence()),
8948                Some(command.shape().to_string()),
8949                Some("workflow completion".to_string()),
8950                "workflow completed before consuming all recorded durable commands",
8951            )));
8952        }
8953        if let Some(sequence) = state
8954            .recorded_continue_as_new_sequence
8955            .filter(|_| !state.continue_as_new_consumed)
8956        {
8957            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8958                "recorded_continue_as_new_unconsumed",
8959                Some(sequence),
8960                Some("continue as new".to_string()),
8961                Some("workflow completion".to_string()),
8962                "workflow completed without consuming its recorded continue-as-new transition",
8963            )));
8964        }
8965        Ok(())
8966    }
8967}
8968
8969fn contiguous_message_stream_count(
8970    pending: &[MessageStreamMessage],
8971    cursor: u64,
8972    max_items: usize,
8973) -> usize {
8974    pending
8975        .iter()
8976        .take(max_items)
8977        .enumerate()
8978        .take_while(|(offset, message)| {
8979            u64::try_from(*offset)
8980                .ok()
8981                .and_then(|offset| cursor.checked_add(offset + 1))
8982                == Some(message.position)
8983        })
8984        .count()
8985}
8986
8987fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
8988    matches!(
8989        event.event_type.as_str(),
8990        "ActivityScheduled"
8991            | "TimerScheduled"
8992            | "ChildWorkflowScheduled"
8993            | "SignalWaitOpened"
8994            | "ConditionWaitOpened"
8995            | "SearchAttributesUpserted"
8996            | "SideEffectRecorded"
8997            | "VersionMarkerRecorded"
8998            | "MemoUpserted"
8999            | "WorkflowContinuedAsNew"
9000    )
9001}
9002
9003#[derive(Debug)]
9004struct WorkflowState {
9005    workflow_id: Option<String>,
9006    run_id: Option<String>,
9007    task_queue: String,
9008    payload_codec: String,
9009    history_events: Arc<Vec<HistoryEvent>>,
9010    history_budget: WorkflowHistoryBudget,
9011    cancel_requested: bool,
9012    resume_signal: Option<ResumeSignal>,
9013    recorded_commands: Vec<RecordedCommand>,
9014    selection_markers: Vec<SelectionMarker>,
9015    selection_marker_cursor: usize,
9016    cancelled_selection_members: Vec<SelectionCancellation>,
9017    recorded_continue_as_new_sequence: Option<u64>,
9018    continue_as_new_consumed: bool,
9019    command_cursor: usize,
9020    condition_wait_occurrence_counter: u64,
9021    matched_recorded_pending: bool,
9022    version_markers: HashMap<String, (i32, u64)>,
9023    workflow_command_identity: String,
9024    workflow_stream_command_counter: u64,
9025    commands: Vec<Value>,
9026    message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9027    message_stream_cursors: HashMap<String, u64>,
9028    message_stream_waits: HashMap<String, u64>,
9029}
9030
9031impl WorkflowState {
9032    #[cfg(test)]
9033    fn new(
9034        history: Vec<HistoryEvent>,
9035        task_queue: String,
9036        payload_codec: String,
9037        resume_signal: Option<ResumeSignal>,
9038    ) -> Result<Self> {
9039        Self::new_with_identity(
9040            history,
9041            None,
9042            None,
9043            task_queue,
9044            payload_codec,
9045            resume_signal,
9046        )
9047    }
9048
9049    fn new_with_identity(
9050        history: Vec<HistoryEvent>,
9051        workflow_id: Option<String>,
9052        run_id: Option<String>,
9053        task_queue: String,
9054        payload_codec: String,
9055        resume_signal: Option<ResumeSignal>,
9056    ) -> Result<Self> {
9057        let recorded_commands = recorded_commands(
9058            &history,
9059            &payload_codec,
9060            WorkflowIdentity {
9061                workflow_id: workflow_id.clone(),
9062                run_id: run_id.clone(),
9063            },
9064        )?;
9065        let selection_markers = recorded_selection_markers(&history)?;
9066        let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9067        let recorded_continue_as_new = history
9068            .iter()
9069            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9070            .collect::<Vec<_>>();
9071        if recorded_continue_as_new.len() > 1 {
9072            return Err(invalid_recorded_history(
9073                "duplicate_continue_as_new_transition",
9074                recorded_continue_as_new
9075                    .last()
9076                    .and_then(|event| durable_event_sequence(event))
9077                    .unwrap_or(0),
9078                "one WorkflowContinuedAsNew event",
9079                &format!(
9080                    "{} WorkflowContinuedAsNew events",
9081                    recorded_continue_as_new.len()
9082                ),
9083                "workflow history records one continue-as-new transition more than once",
9084            ));
9085        }
9086        let recorded_continue_as_new_sequence = recorded_continue_as_new
9087            .first()
9088            .map(|event| {
9089                durable_event_sequence(event).ok_or_else(|| {
9090                    Error::NonDeterministicReplay(ReplayFailure::new(
9091                        "continue_as_new_sequence_missing",
9092                        None,
9093                        Some("recorded transition sequence".to_string()),
9094                        Some("missing sequence".to_string()),
9095                        "WorkflowContinuedAsNew history is missing its recorded sequence",
9096                    ))
9097                })
9098            })
9099            .transpose()?;
9100        let mut message_stream_cursors = HashMap::new();
9101        for event in &history {
9102            if !matches!(
9103                event.event_type.as_str(),
9104                "SignalReceived" | "SignalApplied"
9105            ) || event.payload.get("signal_name").and_then(Value::as_str)
9106                != Some(MESSAGE_STREAM_SIGNAL)
9107            {
9108                continue;
9109            }
9110            let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9111            if arguments.len() != 1 {
9112                continue;
9113            }
9114            let envelope = arguments[0].clone().into_json()?;
9115            let Some(envelope) = envelope.as_object() else {
9116                continue;
9117            };
9118            if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9119            {
9120                continue;
9121            }
9122            let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9123                continue;
9124            };
9125            let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9126            else {
9127                continue;
9128            };
9129            let cursor = message_stream_cursors
9130                .entry(stream_name.to_string())
9131                .or_insert(0);
9132            *cursor = (*cursor).max(through_position);
9133        }
9134        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9135        let cancel_requested = history.iter().any(|event| {
9136            matches!(
9137                event.event_type.as_str(),
9138                "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9139            )
9140        });
9141        Ok(Self {
9142            workflow_command_identity: String::new(),
9143            workflow_stream_command_counter: 0,
9144            workflow_id,
9145            run_id,
9146            task_queue,
9147            payload_codec,
9148            history_events: Arc::new(history),
9149            history_budget: WorkflowHistoryBudget {
9150                event_count,
9151                ..WorkflowHistoryBudget::default()
9152            },
9153            cancel_requested,
9154            resume_signal,
9155            recorded_commands,
9156            selection_markers,
9157            selection_marker_cursor: 0,
9158            cancelled_selection_members,
9159            recorded_continue_as_new_sequence,
9160            continue_as_new_consumed: false,
9161            command_cursor: 0,
9162            condition_wait_occurrence_counter: 0,
9163            matched_recorded_pending: false,
9164            version_markers: HashMap::new(),
9165            commands: Vec::new(),
9166            message_stream_messages: HashMap::new(),
9167            message_stream_cursors,
9168            message_stream_waits: HashMap::new(),
9169        })
9170    }
9171}
9172
9173enum MessageStreamDelivery {
9174    Message(MessageStreamMessage),
9175    Cursor {
9176        stream_name: String,
9177        through_position: u64,
9178    },
9179}
9180
9181fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9182    if arguments.len() != 1 {
9183        return Ok(None);
9184    }
9185    let envelope = arguments
9186        .into_iter()
9187        .next()
9188        .expect("one argument was checked");
9189    let Some(envelope) = envelope.as_object() else {
9190        return Ok(None);
9191    };
9192    let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9193        return Ok(None);
9194    };
9195    if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9196        let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9197        else {
9198            return Ok(None);
9199        };
9200        return Ok(Some(MessageStreamDelivery::Cursor {
9201            stream_name: stream_name.to_string(),
9202            through_position,
9203        }));
9204    }
9205    if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9206        return Ok(None);
9207    }
9208    let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9209        return Ok(None);
9210    };
9211    let Some(position) = envelope
9212        .get("position")
9213        .and_then(Value::as_u64)
9214        .filter(|value| *value > 0)
9215    else {
9216        return Ok(None);
9217    };
9218    let Some(payload_envelope) = envelope.get("payload_envelope") else {
9219        return Ok(None);
9220    };
9221    let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9222    else {
9223        return Ok(None);
9224    };
9225    let decoded = decode_avro_value(&payload_envelope)?;
9226    let AvroValue::Array(values) = decoded else {
9227        return Ok(None);
9228    };
9229    Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9230        stream_name: stream_name.to_string(),
9231        message_id: message_id.to_string(),
9232        position,
9233        arguments: values,
9234    })))
9235}
9236
9237#[derive(Clone, Debug)]
9238enum RecordedCommand {
9239    Activity {
9240        sequence: u64,
9241        activity_type: Option<String>,
9242        options: Option<RecordedActivityOptions>,
9243        outcome: Option<ActivityOutcome>,
9244        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9245    },
9246    Timer {
9247        sequence: u64,
9248        delay_seconds: u64,
9249        fired: bool,
9250        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9251    },
9252    ChildWorkflow {
9253        sequence: u64,
9254        workflow_type: Option<String>,
9255        outcome: Option<ChildWorkflowOutcome>,
9256        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9257    },
9258    SignalWait {
9259        sequence: u64,
9260        signal_name: String,
9261        value: Option<Vec<AvroValue>>,
9262        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9263    },
9264    ConditionWait {
9265        sequence: u64,
9266        occurrence_id: String,
9267        condition_key: Option<String>,
9268        predicate_identity: String,
9269        timeout_seconds: Option<u64>,
9270        result: Option<ConditionWaitResult>,
9271        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9272    },
9273    SearchAttributes {
9274        sequence: u64,
9275        attributes: Value,
9276        attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9277    },
9278    SideEffect {
9279        sequence: u64,
9280        value: AvroValue,
9281    },
9282    VersionMarker {
9283        sequence: u64,
9284        change_id: String,
9285        version: i32,
9286    },
9287    Memo {
9288        sequence: u64,
9289        entries: AvroValue,
9290    },
9291}
9292
9293#[derive(Clone, Debug, PartialEq, Eq)]
9294struct SelectionMarker {
9295    selection_group_id: String,
9296    selection_group_base_sequence: u64,
9297    selection_group_size: usize,
9298    member_key: SelectionKey,
9299    member_index: usize,
9300    member_base_sequence: u64,
9301    member_size: usize,
9302    operation_kind: String,
9303    operation_identity: String,
9304    outcome: String,
9305    resolution_event_id: String,
9306    resolution_event_type: String,
9307}
9308
9309#[derive(Clone, Debug, PartialEq, Eq)]
9310struct SelectionCancellation {
9311    selection_group_id: String,
9312    member_key: SelectionKey,
9313    member_index: usize,
9314    member_base_sequence: u64,
9315    member_size: usize,
9316    operation_kind: String,
9317    operation_identity: String,
9318}
9319
9320fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9321    let mut markers: Vec<SelectionMarker> = Vec::new();
9322    for event in events
9323        .iter()
9324        .filter(|event| event.event_type == "SelectionResolved")
9325    {
9326        let payload = &event.payload;
9327        let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9328        let group_size = required_selection_usize(payload, "selection_group_size")?;
9329        let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9330        let member_size = required_selection_usize(payload, "member_size")?;
9331        let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9332        let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9333            invalid_recorded_history(
9334                "selection_marker_invalid",
9335                base_sequence,
9336                "non-empty selection_group_id",
9337                &payload.to_string(),
9338                "selection winner history is missing its durable group identity",
9339            )
9340        })?;
9341        let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9342        if group_id != expected_group_id {
9343            return Err(invalid_recorded_history(
9344                "selection_marker_invalid",
9345                base_sequence,
9346                &expected_group_id,
9347                &group_id,
9348                "selection winner history contains an incompatible group identity",
9349            ));
9350        }
9351        let group_end = base_sequence
9352            .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9353            .unwrap_or(u64::MAX);
9354        let member_end = member_base_sequence
9355            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9356            .unwrap_or(u64::MAX);
9357        if member_index >= group_size
9358            || member_base_sequence < base_sequence
9359            || member_end > group_end
9360        {
9361            return Err(invalid_recorded_history(
9362                "selection_marker_invalid",
9363                base_sequence,
9364                "winner member within selection group bounds",
9365                &payload.to_string(),
9366                "selection winner history contains an invalid member range",
9367            ));
9368        }
9369        let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9370            invalid_recorded_history(
9371                "selection_marker_invalid",
9372                base_sequence,
9373                "selection operation kind",
9374                &payload.to_string(),
9375                "selection winner history is missing its operation kind",
9376            )
9377        })?;
9378        if !matches!(
9379            operation_kind.as_str(),
9380            "activity" | "child" | "timer" | "signal" | "condition" | "group"
9381        ) {
9382            return Err(invalid_recorded_history(
9383                "selection_marker_invalid",
9384                base_sequence,
9385                "activity, child, timer, signal, condition, or group",
9386                &operation_kind,
9387                "selection winner history contains an unsupported operation kind",
9388            ));
9389        }
9390        let operation_identity =
9391            payload_string(payload, "operation_identity").ok_or_else(|| {
9392                invalid_recorded_history(
9393                    "selection_marker_invalid",
9394                    base_sequence,
9395                    "non-empty operation identity",
9396                    &payload.to_string(),
9397                    "selection winner history is missing its durable operation identity",
9398                )
9399            })?;
9400        let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9401            invalid_recorded_history(
9402                "selection_marker_invalid",
9403                base_sequence,
9404                "completed or failed selection outcome",
9405                &payload.to_string(),
9406                "selection winner history is missing its outcome",
9407            )
9408        })?;
9409        if !matches!(outcome.as_str(), "completed" | "failed") {
9410            return Err(invalid_recorded_history(
9411                "selection_marker_invalid",
9412                base_sequence,
9413                "completed or failed selection outcome",
9414                &outcome,
9415                "selection winner history contains an unsupported outcome",
9416            ));
9417        }
9418        let marker = SelectionMarker {
9419            selection_group_id: group_id,
9420            selection_group_base_sequence: base_sequence,
9421            selection_group_size: group_size,
9422            member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9423            member_index,
9424            member_base_sequence,
9425            member_size,
9426            operation_kind,
9427            operation_identity,
9428            outcome,
9429            resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9430                || {
9431                    invalid_recorded_history(
9432                        "selection_marker_invalid",
9433                        base_sequence,
9434                        "durable resolution_event_id",
9435                        &payload.to_string(),
9436                        "selection winner history is missing its terminal event identity",
9437                    )
9438                },
9439            )?,
9440            resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9441                || {
9442                    invalid_recorded_history(
9443                        "selection_marker_invalid",
9444                        base_sequence,
9445                        "durable resolution_event_type",
9446                        &payload.to_string(),
9447                        "selection winner history is missing its terminal event type",
9448                    )
9449                },
9450            )?,
9451        };
9452        if let Some(existing) = markers
9453            .iter()
9454            .find(|existing| existing.selection_group_id == marker.selection_group_id)
9455        {
9456            if existing != &marker {
9457                return Err(invalid_recorded_history(
9458                    "selection_marker_conflict",
9459                    base_sequence,
9460                    &format!("one winner for {}", marker.selection_group_id),
9461                    &payload.to_string(),
9462                    "selection history records conflicting winners for one durable group",
9463                ));
9464            }
9465            continue;
9466        }
9467        markers.push(marker);
9468    }
9469    Ok(markers)
9470}
9471
9472fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9473    let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9474    for event in events
9475        .iter()
9476        .filter(|event| event.event_type == "SelectionOperationCancelled")
9477    {
9478        let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9479            invalid_recorded_history(
9480                "selection_cancellation_invalid",
9481                0,
9482                "non-empty selection_group_id",
9483                &event.payload.to_string(),
9484                "selection cancellation history is missing its group identity",
9485            )
9486        })?;
9487        let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9488        let marker = SelectionCancellation {
9489            selection_group_id: group_id,
9490            member_key: selection_key_from_value(
9491                event.payload.get("member_key"),
9492                member_base_sequence,
9493            )?,
9494            member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9495            member_base_sequence,
9496            member_size: required_selection_usize(&event.payload, "member_size")?,
9497            operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9498                invalid_recorded_history(
9499                    "selection_cancellation_invalid",
9500                    member_base_sequence,
9501                    "selection operation kind",
9502                    &event.payload.to_string(),
9503                    "selection cancellation is missing its operation kind",
9504                )
9505            })?,
9506            operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9507                || {
9508                    invalid_recorded_history(
9509                        "selection_cancellation_invalid",
9510                        member_base_sequence,
9511                        "selection operation identity",
9512                        &event.payload.to_string(),
9513                        "selection cancellation is missing its operation identity",
9514                    )
9515                },
9516            )?,
9517        };
9518        if let Some(existing) = cancelled.iter().find(|recorded| {
9519            recorded.selection_group_id == marker.selection_group_id
9520                && recorded.member_base_sequence == marker.member_base_sequence
9521        }) {
9522            if existing != &marker {
9523                return Err(invalid_recorded_history(
9524                    "selection_cancellation_conflict",
9525                    member_base_sequence,
9526                    "one stable SelectionOperationCancelled marker",
9527                    &event.payload.to_string(),
9528                    "selection cancellation history contains conflicting member metadata",
9529                ));
9530            }
9531        } else {
9532            cancelled.push(marker);
9533        }
9534    }
9535    Ok(cancelled)
9536}
9537
9538fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9539    payload
9540        .get(field)
9541        .and_then(value_as_u64)
9542        .filter(|value| *value > 0)
9543        .ok_or_else(|| {
9544            invalid_recorded_history(
9545                "selection_marker_invalid",
9546                0,
9547                &format!("positive integer {field}"),
9548                &payload.to_string(),
9549                "selection history contains invalid durable identity metadata",
9550            )
9551        })
9552}
9553
9554fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9555    required_selection_usize_allow_zero(payload, field).and_then(|value| {
9556        if value > 0 {
9557            Ok(value)
9558        } else {
9559            Err(invalid_recorded_history(
9560                "selection_marker_invalid",
9561                0,
9562                &format!("positive integer {field}"),
9563                &payload.to_string(),
9564                "selection history contains invalid durable identity metadata",
9565            ))
9566        }
9567    })
9568}
9569
9570fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9571    payload
9572        .get(field)
9573        .and_then(value_as_u64)
9574        .and_then(|value| usize::try_from(value).ok())
9575        .ok_or_else(|| {
9576            invalid_recorded_history(
9577                "selection_marker_invalid",
9578                0,
9579                &format!("non-negative integer {field}"),
9580                &payload.to_string(),
9581                "selection history contains invalid durable identity metadata",
9582            )
9583        })
9584}
9585
9586#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9587struct RecordedActivityOptions {
9588    task_queue: RecordedSnapshotValue<Option<String>>,
9589    execution_mode: RecordedSnapshotValue<Option<String>>,
9590    retry_policy: ActivityRetrySnapshot,
9591}
9592
9593#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9594enum RecordedSnapshotValue<T> {
9595    /// Older history did not persist this field, so it cannot constrain replay.
9596    Unknown,
9597    Known(T),
9598}
9599
9600impl<T: PartialEq> RecordedSnapshotValue<T> {
9601    fn matches_current(&self, current: &Self) -> bool {
9602        match self {
9603            Self::Unknown => true,
9604            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9605        }
9606    }
9607}
9608
9609#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9610struct ActivityRetrySnapshot {
9611    snapshot_version: RecordedSnapshotValue<Option<u64>>,
9612    max_attempts: RecordedSnapshotValue<Option<u64>>,
9613    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9614    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9615    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9616    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9617    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9618    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9619}
9620
9621impl ActivityRetrySnapshot {
9622    fn matches_current(&self, current: &Self) -> bool {
9623        self.snapshot_version
9624            .matches_current(&current.snapshot_version)
9625            && self.max_attempts.matches_current(&current.max_attempts)
9626            && self
9627                .backoff_seconds
9628                .matches_current(&current.backoff_seconds)
9629            && self
9630                .start_to_close_timeout
9631                .matches_current(&current.start_to_close_timeout)
9632            && self
9633                .schedule_to_start_timeout
9634                .matches_current(&current.schedule_to_start_timeout)
9635            && self
9636                .schedule_to_close_timeout
9637                .matches_current(&current.schedule_to_close_timeout)
9638            && self
9639                .heartbeat_timeout
9640                .matches_current(&current.heartbeat_timeout)
9641            && self
9642                .non_retryable_error_types
9643                .matches_current(&current.non_retryable_error_types)
9644    }
9645}
9646
9647fn recorded_optional_u64(
9648    object: Option<&serde_json::Map<String, Value>>,
9649    field: &str,
9650) -> RecordedSnapshotValue<Option<u64>> {
9651    match object.and_then(|object| object.get(field)) {
9652        None => RecordedSnapshotValue::Unknown,
9653        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9654        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9655    }
9656}
9657
9658fn recorded_optional_string(
9659    object: &serde_json::Map<String, Value>,
9660    field: &str,
9661) -> RecordedSnapshotValue<Option<String>> {
9662    match object.get(field) {
9663        None => RecordedSnapshotValue::Unknown,
9664        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9665        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9666    }
9667}
9668
9669fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9670    let policy = policy.and_then(Value::as_object);
9671    let backoff_seconds = policy
9672        .and_then(|policy| policy.get("backoff_seconds"))
9673        .and_then(Value::as_array)
9674        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9675        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9676    let mut non_retryable_error_types = Vec::new();
9677    for error_type in policy
9678        .and_then(|policy| policy.get("non_retryable_error_types"))
9679        .and_then(Value::as_array)
9680        .into_iter()
9681        .flatten()
9682        .filter_map(Value::as_str)
9683        .map(str::trim)
9684        .filter(|error_type| !error_type.is_empty())
9685    {
9686        if !non_retryable_error_types
9687            .iter()
9688            .any(|recorded| recorded == error_type)
9689        {
9690            non_retryable_error_types.push(error_type.to_string());
9691        }
9692    }
9693
9694    ActivityRetrySnapshot {
9695        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9696        max_attempts: recorded_optional_u64(policy, "max_attempts"),
9697        backoff_seconds,
9698        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9699        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9700        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9701        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9702        non_retryable_error_types: if policy
9703            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9704        {
9705            RecordedSnapshotValue::Known(non_retryable_error_types)
9706        } else {
9707            RecordedSnapshotValue::Unknown
9708        },
9709    }
9710}
9711
9712fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9713    let policy = options.retry_policy.as_ref();
9714    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9715        Some(Value::Null) => None,
9716        Some(value) => value_as_u64(value),
9717        None => Some(1),
9718    };
9719    let backoff_seconds = policy
9720        .and_then(|policy| policy.get("backoff_seconds"))
9721        .and_then(Value::as_array)
9722        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9723        .unwrap_or_default();
9724    let non_retryable_error_types = policy
9725        .and_then(|policy| policy.get("non_retryable_error_types"))
9726        .and_then(Value::as_array)
9727        .into_iter()
9728        .flatten()
9729        .filter_map(Value::as_str)
9730        .map(str::to_string)
9731        .collect();
9732
9733    ActivityRetrySnapshot {
9734        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9735        max_attempts: RecordedSnapshotValue::Known(max_attempts),
9736        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9737        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9738        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9739        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9740        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9741        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9742    }
9743}
9744
9745fn activity_options_description(options: &RecordedActivityOptions) -> String {
9746    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9747}
9748
9749impl RecordedCommand {
9750    fn sequence(&self) -> u64 {
9751        match self {
9752            Self::Activity { sequence, .. }
9753            | Self::Timer { sequence, .. }
9754            | Self::ChildWorkflow { sequence, .. }
9755            | Self::SignalWait { sequence, .. }
9756            | Self::ConditionWait { sequence, .. }
9757            | Self::SearchAttributes { sequence, .. }
9758            | Self::SideEffect { sequence, .. }
9759            | Self::VersionMarker { sequence, .. }
9760            | Self::Memo { sequence, .. } => *sequence,
9761        }
9762    }
9763
9764    fn shape(&self) -> &'static str {
9765        match self {
9766            Self::Activity { .. } => "activity",
9767            Self::Timer { .. } => "timer",
9768            Self::ChildWorkflow { .. } => "child workflow",
9769            Self::SignalWait { .. } => "signal wait",
9770            Self::ConditionWait { .. } => "condition wait",
9771            Self::SearchAttributes { .. } => "search-attribute update",
9772            Self::SideEffect { .. } => "side effect",
9773            Self::VersionMarker { .. } => "version marker",
9774            Self::Memo { .. } => "memo upsert",
9775        }
9776    }
9777}
9778
9779fn ensure_version_supported(
9780    change_id: &str,
9781    version: i32,
9782    min_supported: i32,
9783    max_supported: i32,
9784    sequence: u64,
9785) -> Result<()> {
9786    if (min_supported..=max_supported).contains(&version) {
9787        return Ok(());
9788    }
9789    Err(Error::NonDeterministicReplay(ReplayFailure::new(
9790        "version_marker_incompatible_range",
9791        (sequence != 0).then_some(sequence),
9792        Some(format!("{min_supported}..={max_supported}")),
9793        Some(format!("{change_id}:{version}")),
9794        "recorded workflow version is outside the range supported by current code",
9795    )))
9796}
9797
9798#[derive(Clone, Debug)]
9799struct ResumeSignal {
9800    signal_name: String,
9801    arguments: Vec<AvroValue>,
9802}
9803
9804const MAX_PARALLEL_OPERATIONS: usize = 1000;
9805
9806fn parallel_group_prefix(kind: &str) -> &'static str {
9807    match kind {
9808        "activity" => "parallel-activities",
9809        "child" => "parallel-children",
9810        "timer" => "parallel-timers",
9811        _ => "parallel-calls",
9812    }
9813}
9814
9815fn parallel_group_entry(
9816    base_sequence: u64,
9817    size: usize,
9818    index: usize,
9819    kind: &str,
9820) -> ParallelGroupMetadata {
9821    ParallelGroupMetadata {
9822        parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9823        parallel_group_kind: kind.to_string(),
9824        parallel_group_base_sequence: base_sequence,
9825        parallel_group_size: size,
9826        parallel_group_index: index,
9827        parallel_group_mode: None,
9828        selection_member_key: None,
9829        selection_member_index: None,
9830        selection_member_base_sequence: None,
9831        selection_member_size: None,
9832        selection_member_kind: None,
9833    }
9834}
9835
9836struct SelectionMemberMetadata {
9837    key: SelectionKey,
9838    index: usize,
9839    base_sequence: u64,
9840    size: usize,
9841    kind: String,
9842}
9843
9844fn selection_group_entry(
9845    base_sequence: u64,
9846    size: usize,
9847    index: usize,
9848    kind: &str,
9849    member: &SelectionMemberMetadata,
9850) -> ParallelGroupMetadata {
9851    ParallelGroupMetadata {
9852        parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9853        parallel_group_kind: kind.to_string(),
9854        parallel_group_base_sequence: base_sequence,
9855        parallel_group_size: size,
9856        parallel_group_index: index,
9857        parallel_group_mode: Some("select".to_string()),
9858        selection_member_key: Some(member.key.clone()),
9859        selection_member_index: Some(member.index),
9860        selection_member_base_sequence: Some(member.base_sequence),
9861        selection_member_size: Some(member.size),
9862        selection_member_kind: Some(member.kind.clone()),
9863    }
9864}
9865
9866fn apply_parallel_group_path(
9867    command: &mut serde_json::Map<String, Value>,
9868    path: &[ParallelGroupMetadata],
9869) {
9870    let Some(inner) = path.last() else {
9871        return;
9872    };
9873    command.insert(
9874        "parallel_group_id".to_string(),
9875        json!(inner.parallel_group_id),
9876    );
9877    command.insert(
9878        "parallel_group_kind".to_string(),
9879        json!(inner.parallel_group_kind),
9880    );
9881    command.insert(
9882        "parallel_group_base_sequence".to_string(),
9883        json!(inner.parallel_group_base_sequence),
9884    );
9885    command.insert(
9886        "parallel_group_size".to_string(),
9887        json!(inner.parallel_group_size),
9888    );
9889    command.insert(
9890        "parallel_group_index".to_string(),
9891        json!(inner.parallel_group_index),
9892    );
9893    if let Some(mode) = &inner.parallel_group_mode {
9894        command.insert("parallel_group_mode".to_string(), json!(mode));
9895    }
9896    if let Some(key) = &inner.selection_member_key {
9897        command.insert("selection_member_key".to_string(), json!(key));
9898    }
9899    if let Some(index) = inner.selection_member_index {
9900        command.insert("selection_member_index".to_string(), json!(index));
9901    }
9902    if let Some(base_sequence) = inner.selection_member_base_sequence {
9903        command.insert(
9904            "selection_member_base_sequence".to_string(),
9905            json!(base_sequence),
9906        );
9907    }
9908    if let Some(size) = inner.selection_member_size {
9909        command.insert("selection_member_size".to_string(), json!(size));
9910    }
9911    if let Some(kind) = &inner.selection_member_kind {
9912        command.insert("selection_member_kind".to_string(), json!(kind));
9913    }
9914    command.insert("parallel_group_path".to_string(), json!(path));
9915}
9916
9917fn ensure_parallel_path_matches(
9918    sequence: u64,
9919    recorded: Option<&[ParallelGroupMetadata]>,
9920    expected: &[ParallelGroupMetadata],
9921) -> Result<()> {
9922    match (recorded, expected.is_empty()) {
9923        (None, true) => Ok(()),
9924        (Some(recorded), false) if recorded == expected => Ok(()),
9925        (None, false) => Err(invalid_recorded_history(
9926            "parallel_group_metadata_missing",
9927            sequence,
9928            &serde_json::to_string(expected).unwrap_or_default(),
9929            "<missing>",
9930            "recorded parallel member is missing its durable group path",
9931        )),
9932        (Some(recorded), true) => Err(invalid_recorded_history(
9933            "parallel_group_shape_mismatch",
9934            sequence,
9935            "sequential command",
9936            &serde_json::to_string(recorded).unwrap_or_default(),
9937            "recorded command belonged to a parallel group but current code schedules it sequentially",
9938        )),
9939        (Some(recorded), false) => Err(invalid_recorded_history(
9940            "parallel_group_shape_mismatch",
9941            sequence,
9942            &serde_json::to_string(recorded).unwrap_or_default(),
9943            &serde_json::to_string(expected).unwrap_or_default(),
9944            "recorded parallel-group identity or path changed during replay",
9945        )),
9946    }
9947}
9948
9949#[derive(Clone, Debug)]
9950enum ParallelShape {
9951    Leaf,
9952    Group(Vec<ParallelShape>),
9953}
9954
9955struct ParallelDescriptor {
9956    operation: ParallelOperation,
9957    offset: usize,
9958    member_path: Vec<usize>,
9959    group_path: Vec<ParallelGroupMetadata>,
9960}
9961
9962fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9963    operations
9964        .iter()
9965        .map(|operation| match operation {
9966            ParallelOperation::Group(children) => parallel_leaf_count(children),
9967            _ => 1,
9968        })
9969        .sum()
9970}
9971
9972fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
9973    match operation {
9974        ParallelOperation::Activity { .. } => Some("activity"),
9975        ParallelOperation::ChildWorkflow { .. } => Some("child"),
9976        ParallelOperation::Timer(_) => Some("timer"),
9977        ParallelOperation::Signal(_) => Some("signal"),
9978        ParallelOperation::Condition { .. } => Some("condition"),
9979        ParallelOperation::Group(children) => parallel_group_kind(children),
9980    }
9981}
9982
9983fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
9984    let mut kind = None;
9985    for operation in operations {
9986        let Some(operation_kind) = parallel_operation_kind(operation) else {
9987            continue;
9988        };
9989        match kind {
9990            None => kind = Some(operation_kind),
9991            Some(current) if current == operation_kind => {}
9992            Some(_) => return Some("mixed"),
9993        }
9994    }
9995    kind
9996}
9997
9998fn validate_parallel_operations(
9999    operations: &[ParallelOperation],
10000    member_path: &mut Vec<usize>,
10001    root: bool,
10002) -> Result<()> {
10003    let leaves = parallel_leaf_count(operations);
10004    if leaves > MAX_PARALLEL_OPERATIONS {
10005        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10006            reason: "fan_out_limit_exceeded",
10007            member_path: member_path.clone(),
10008            message: format!(
10009                "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10010            ),
10011        }));
10012    }
10013    if !root && operations.is_empty() {
10014        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10015            reason: "nested_group_empty",
10016            member_path: member_path.clone(),
10017            message: "a nested group must contain at least one durable leaf".to_string(),
10018        }));
10019    }
10020
10021    for (index, operation) in operations.iter().enumerate() {
10022        member_path.push(index);
10023        match operation {
10024            ParallelOperation::Activity {
10025                options, arguments, ..
10026            } => {
10027                options
10028                    .validate()
10029                    .map_err(|error| Error::InvalidActivityOptions(error))?;
10030                if let Err(error) = arguments {
10031                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10032                        reason: "arguments_invalid",
10033                        member_path: member_path.clone(),
10034                        message: error.to_string(),
10035                    }));
10036                }
10037            }
10038            ParallelOperation::ChildWorkflow {
10039                options, arguments, ..
10040            } => {
10041                validate_parallel_child_options(options)?;
10042                if let Err(error) = arguments {
10043                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10044                        reason: "arguments_invalid",
10045                        member_path: member_path.clone(),
10046                        message: error.to_string(),
10047                    }));
10048                }
10049            }
10050            ParallelOperation::Timer(duration)
10051                if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10052            {
10053                return Err(Error::TimerDurationOverflow);
10054            }
10055            ParallelOperation::Timer(_) => {}
10056            ParallelOperation::Signal(signal_name) => {
10057                validate_user_signal_name(signal_name)?;
10058                if signal_name.trim().is_empty() {
10059                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10060                        reason: "signal_name_empty",
10061                        member_path: member_path.clone(),
10062                        message: "signal wait name must not be empty".to_string(),
10063                    }));
10064                }
10065            }
10066            ParallelOperation::Condition { options, .. } => {
10067                options.validate()?;
10068            }
10069            ParallelOperation::Group(children) => {
10070                validate_parallel_operations(children, member_path, false)?;
10071            }
10072        }
10073        member_path.pop();
10074    }
10075    Ok(())
10076}
10077
10078fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10079    if options.task_queue.trim().is_empty() {
10080        return Err(Error::InvalidChildWorkflowOptions(
10081            "task_queue must not be empty".to_string(),
10082        ));
10083    }
10084    for (name, value) in [
10085        (
10086            "execution_timeout_seconds",
10087            options.execution_timeout_seconds,
10088        ),
10089        ("run_timeout_seconds", options.run_timeout_seconds),
10090    ] {
10091        if value == Some(0) {
10092            return Err(Error::InvalidChildWorkflowOptions(format!(
10093                "{name} must be at least 1"
10094            )));
10095        }
10096    }
10097    if options
10098        .retry_policy
10099        .as_ref()
10100        .is_some_and(|policy| policy.max_attempts == Some(0))
10101    {
10102        return Err(Error::InvalidChildWorkflowOptions(
10103            "retry_policy.max_attempts must be at least 1".to_string(),
10104        ));
10105    }
10106    Ok(())
10107}
10108
10109fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10110    ParallelShape::Group(
10111        operations
10112            .iter()
10113            .map(|operation| match operation {
10114                ParallelOperation::Group(children) => parallel_shape(children),
10115                _ => ParallelShape::Leaf,
10116            })
10117            .collect(),
10118    )
10119}
10120
10121fn parallel_descriptors(
10122    operations: Vec<ParallelOperation>,
10123    base_sequence: u64,
10124) -> Result<Vec<ParallelDescriptor>> {
10125    let size = parallel_leaf_count(&operations);
10126    let kind = parallel_group_kind(&operations).unwrap_or("activity");
10127    let mut descriptors = Vec::with_capacity(size);
10128    let mut cursor = 0;
10129
10130    for (index, operation) in operations.into_iter().enumerate() {
10131        match operation {
10132            ParallelOperation::Group(children) => {
10133                let child_base = base_sequence
10134                    .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10135                    .ok_or(Error::TimerDurationOverflow)?;
10136                for mut descriptor in parallel_descriptors(children, child_base)? {
10137                    let outer_index = cursor + descriptor.offset;
10138                    descriptor.group_path.insert(
10139                        0,
10140                        parallel_group_entry(base_sequence, size, outer_index, kind),
10141                    );
10142                    descriptor.member_path.insert(0, index);
10143                    descriptor.offset = outer_index;
10144                    descriptors.push(descriptor);
10145                }
10146                cursor = descriptors.len();
10147            }
10148            operation => {
10149                descriptors.push(ParallelDescriptor {
10150                    operation,
10151                    offset: cursor,
10152                    member_path: vec![index],
10153                    group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10154                });
10155                cursor += 1;
10156            }
10157        }
10158    }
10159    Ok(descriptors)
10160}
10161
10162enum ParallelLeafCall {
10163    Activity(ActivityCall),
10164    ChildWorkflow(ChildWorkflowCall),
10165    Timer(TimerCall),
10166    Signal(SignalCall),
10167    Condition(ConditionWaitCall),
10168}
10169
10170fn parallel_leaf_call(
10171    ctx: &WorkflowContext,
10172    operation: ParallelOperation,
10173    parallel_group_path: Vec<ParallelGroupMetadata>,
10174) -> ParallelLeafCall {
10175    match operation {
10176        ParallelOperation::Activity {
10177            activity_type,
10178            options,
10179            arguments,
10180        } => ParallelLeafCall::Activity(ActivityCall {
10181            ctx: ctx.clone(),
10182            activity_type,
10183            options,
10184            args: Some(arguments),
10185            scheduled: false,
10186            parallel_group_path,
10187        }),
10188        ParallelOperation::ChildWorkflow {
10189            workflow_type,
10190            options,
10191            arguments,
10192        } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10193            ctx: ctx.clone(),
10194            workflow_type,
10195            options,
10196            args: Some(arguments),
10197            scheduled: false,
10198            matched_pending: false,
10199            parallel_group_path,
10200        }),
10201        ParallelOperation::Timer(duration) => {
10202            let delay_seconds = duration
10203                .as_secs()
10204                .checked_add(u64::from(duration.subsec_nanos() > 0));
10205            ParallelLeafCall::Timer(TimerCall {
10206                ctx: ctx.clone(),
10207                delay_seconds,
10208                scheduled: false,
10209                matched_pending: false,
10210                parallel_group_path,
10211            })
10212        }
10213        ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10214            ctx: ctx.clone(),
10215            signal_name,
10216            runtime_reserved_allowed: false,
10217            opened_wait: false,
10218            matched_pending: false,
10219            parallel_group_path,
10220        }),
10221        ParallelOperation::Condition { options, predicate } => {
10222            ParallelLeafCall::Condition(ConditionWaitCall {
10223                ctx: ctx.clone(),
10224                options,
10225                predicate,
10226                occurrence_id: None,
10227                opened_wait: false,
10228                parallel_group_path,
10229            })
10230        }
10231        ParallelOperation::Group(_) => {
10232            unreachable!("parallel descriptors contain only durable leaves")
10233        }
10234    }
10235}
10236
10237impl ParallelLeafCall {
10238    fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10239        match self {
10240            Self::Activity(call) => Pin::new(call)
10241                .poll_avro_value(cx)
10242                .map_ok(ParallelAvroResult::Activity),
10243            Self::ChildWorkflow(call) => Pin::new(call)
10244                .poll_avro_value(cx)
10245                .map_ok(ParallelAvroResult::ChildWorkflow),
10246            Self::Timer(call) => Pin::new(call)
10247                .poll(cx)
10248                .map_ok(|()| ParallelAvroResult::Timer),
10249            Self::Signal(call) => Pin::new(call)
10250                .poll_avro_value(cx)
10251                .map_ok(ParallelAvroResult::Signal),
10252            Self::Condition(call) => Pin::new(call)
10253                .poll(cx)
10254                .map_ok(ParallelAvroResult::Condition),
10255        }
10256    }
10257}
10258
10259struct ParallelLeaf {
10260    call: ParallelLeafCall,
10261    member_path: Vec<usize>,
10262    group_path: Vec<ParallelGroupMetadata>,
10263    result: Option<ParallelAvroResult>,
10264}
10265
10266/// Future returned by [`WorkflowContext::parallel`].
10267pub struct ParallelCall {
10268    ctx: WorkflowContext,
10269    operations: Option<Vec<ParallelOperation>>,
10270    shape: Option<ParallelShape>,
10271    leaves: Vec<ParallelLeaf>,
10272}
10273
10274impl ParallelCall {
10275    fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10276        Self {
10277            ctx,
10278            operations: Some(operations),
10279            shape: None,
10280            leaves: Vec::new(),
10281        }
10282    }
10283
10284    fn initialize(&mut self) -> Result<()> {
10285        let operations = self.operations.take().unwrap_or_default();
10286        validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10287        self.shape = Some(parallel_shape(&operations));
10288        if operations.is_empty() {
10289            return Ok(());
10290        }
10291
10292        let base_sequence = {
10293            let state = self
10294                .ctx
10295                .state
10296                .lock()
10297                .map_err(|_| Error::WorkflowStatePoisoned)?;
10298            if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10299                recorded.sequence()
10300            } else {
10301                let last = state
10302                    .recorded_commands
10303                    .last()
10304                    .map(RecordedCommand::sequence)
10305                    .unwrap_or(0);
10306                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10307                    .and_then(|sequence| sequence.checked_add(1))
10308                    .ok_or_else(|| {
10309                        Error::InvalidParallelGroup(ParallelGroupError {
10310                            reason: "sequence_overflow",
10311                            member_path: Vec::new(),
10312                            message: "parallel group sequence identity overflowed u64".to_string(),
10313                        })
10314                    })?
10315            }
10316        };
10317
10318        self.leaves = parallel_descriptors(operations, base_sequence)?
10319            .into_iter()
10320            .map(|descriptor| {
10321                let call = parallel_leaf_call(
10322                    &self.ctx,
10323                    descriptor.operation,
10324                    descriptor.group_path.clone(),
10325                );
10326                ParallelLeaf {
10327                    call,
10328                    member_path: descriptor.member_path,
10329                    group_path: descriptor.group_path,
10330                    result: None,
10331                }
10332            })
10333            .collect();
10334        Ok(())
10335    }
10336
10337    fn poll_avro_value(
10338        mut self: Pin<&mut Self>,
10339        cx: &mut TaskContext<'_>,
10340    ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10341        if self.operations.is_some() {
10342            if let Err(error) = self.initialize() {
10343                return Poll::Ready(Err(error));
10344            }
10345        }
10346        if self.leaves.is_empty() {
10347            return Poll::Ready(Ok(Vec::new()));
10348        }
10349
10350        let mut failures = Vec::new();
10351        let mut pending = false;
10352        for (index, leaf) in self.leaves.iter_mut().enumerate() {
10353            if leaf.result.is_some() {
10354                continue;
10355            }
10356            match leaf.call.poll_avro_value(cx) {
10357                Poll::Ready(Ok(result)) => leaf.result = Some(result),
10358                Poll::Ready(Err(error)) => failures.push((index, error)),
10359                Poll::Pending => pending = true,
10360            }
10361        }
10362
10363        if !failures.is_empty() {
10364            if let Some(position) = failures
10365                .iter()
10366                .position(|(_, error)| workflow_task_integrity_error(error))
10367            {
10368                return Poll::Ready(Err(failures.remove(position).1));
10369            }
10370            failures.sort_by_key(|(index, _)| *index);
10371            let (failed_index, cause) = failures.remove(0);
10372            let failed = &self.leaves[failed_index];
10373            let completed = self
10374                .leaves
10375                .iter()
10376                .filter_map(|leaf| {
10377                    leaf.result
10378                        .clone()
10379                        .and_then(|result| result.into_json_result().ok())
10380                        .map(|result| ParallelCompletion {
10381                            member_path: leaf.member_path.clone(),
10382                            result,
10383                        })
10384                })
10385                .collect();
10386            let group_id = failed
10387                .group_path
10388                .first()
10389                .map(|entry| entry.parallel_group_id.clone())
10390                .unwrap_or_default();
10391            return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10392                group_id,
10393                member_path: failed.member_path.clone(),
10394                group_path: failed.group_path.clone(),
10395                completed,
10396                cause: Box::new(cause),
10397            })));
10398        }
10399        if pending {
10400            return Poll::Pending;
10401        }
10402
10403        let mut flat_results = self
10404            .leaves
10405            .iter_mut()
10406            .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10407            .collect::<Vec<_>>()
10408            .into_iter();
10409        let results = parallel_results_for_shape(
10410            self.shape.as_ref().expect("initialized parallel shape"),
10411            &mut flat_results,
10412        );
10413        Poll::Ready(Ok(match results {
10414            ParallelAvroResult::Group(results) => results,
10415            ParallelAvroResult::Activity(_)
10416            | ParallelAvroResult::ChildWorkflow(_)
10417            | ParallelAvroResult::Timer
10418            | ParallelAvroResult::Signal(_)
10419            | ParallelAvroResult::Condition(_) => {
10420                unreachable!("root parallel shape is a group")
10421            }
10422        }))
10423    }
10424}
10425
10426fn parallel_results_for_shape(
10427    shape: &ParallelShape,
10428    flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10429) -> ParallelAvroResult {
10430    match shape {
10431        ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10432        ParallelShape::Group(children) => ParallelAvroResult::Group(
10433            children
10434                .iter()
10435                .map(|child| parallel_results_for_shape(child, flat_results))
10436                .collect(),
10437        ),
10438    }
10439}
10440
10441impl Future for ParallelCall {
10442    type Output = Result<Vec<ParallelResult>>;
10443
10444    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10445        self.poll_avro_value(cx)
10446            .map_ok(|results| {
10447                results
10448                    .into_iter()
10449                    .map(ParallelAvroResult::into_json_result)
10450                    .collect::<Result<Vec<_>>>()
10451            })
10452            .map_ok(|result| result)
10453            .flatten_result()
10454    }
10455}
10456
10457#[derive(Clone, Debug)]
10458struct SelectionMemberPlan {
10459    key: SelectionKey,
10460    index: usize,
10461    base_sequence: u64,
10462    size: usize,
10463    kind: String,
10464    shape: ParallelShape,
10465    leaf_start: usize,
10466}
10467
10468fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10469    match operation {
10470        ParallelOperation::Activity { .. } => "activity",
10471        ParallelOperation::ChildWorkflow { .. } => "child",
10472        ParallelOperation::Timer(_) => "timer",
10473        ParallelOperation::Signal(_) => "signal",
10474        ParallelOperation::Condition { .. } => "condition",
10475        ParallelOperation::Group(_) => "group",
10476    }
10477}
10478
10479fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10480    match operation {
10481        ParallelOperation::Group(children) => parallel_shape(children),
10482        _ => ParallelShape::Leaf,
10483    }
10484}
10485
10486fn selection_descriptors(
10487    operations: Vec<(SelectionKey, ParallelOperation)>,
10488    base_sequence: u64,
10489) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10490    if operations.is_empty() {
10491        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10492            reason: "selection_empty",
10493            member_path: Vec::new(),
10494            message: "durable selection requires at least one operation".to_string(),
10495        }));
10496    }
10497    let operation_refs = operations
10498        .iter()
10499        .map(|(_, operation)| operation)
10500        .collect::<Vec<_>>();
10501    let total_size = operation_refs
10502        .iter()
10503        .map(|operation| match operation {
10504            ParallelOperation::Group(children) => parallel_leaf_count(children),
10505            _ => 1,
10506        })
10507        .sum::<usize>();
10508    if total_size > MAX_PARALLEL_OPERATIONS {
10509        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10510            reason: "fan_out_limit_exceeded",
10511            member_path: Vec::new(),
10512            message: format!(
10513                "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10514            ),
10515        }));
10516    }
10517    let group_kind = {
10518        let mut kind = None;
10519        for operation in &operation_refs {
10520            let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10521            match kind {
10522                None => kind = Some(operation_kind),
10523                Some(current) if current == operation_kind => {}
10524                Some(_) => {
10525                    kind = Some("mixed");
10526                    break;
10527                }
10528            }
10529        }
10530        kind.unwrap_or("mixed")
10531    };
10532
10533    let mut descriptors = Vec::with_capacity(total_size);
10534    let mut members = Vec::with_capacity(operations.len());
10535    let mut cursor = 0usize;
10536    let mut seen_keys: Vec<SelectionKey> = Vec::new();
10537    for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10538        if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10539            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10540                reason: "selection_key_invalid",
10541                member_path: vec![member_index],
10542                message: "selection member keys must be non-empty strings or non-negative integers"
10543                    .to_string(),
10544            }));
10545        }
10546        if seen_keys.contains(&key) {
10547            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10548                reason: "selection_key_duplicate",
10549                member_path: vec![member_index],
10550                message: format!("selection member key {key:?} is duplicated"),
10551            }));
10552        }
10553        seen_keys.push(key.clone());
10554        let member_size = match &operation {
10555            ParallelOperation::Group(children) => parallel_leaf_count(children),
10556            _ => 1,
10557        };
10558        if member_size == 0 {
10559            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10560                reason: "selection_member_empty",
10561                member_path: vec![member_index],
10562                message: "a selection member must contain at least one durable leaf".to_string(),
10563            }));
10564        }
10565        let member_base = base_sequence
10566            .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10567            .ok_or(Error::TimerDurationOverflow)?;
10568        let member_kind = selection_operation_kind(&operation).to_string();
10569        let member_shape = selection_operation_shape(&operation);
10570        let leaf_start = descriptors.len();
10571        match operation {
10572            ParallelOperation::Group(children) => {
10573                validate_parallel_operations(&children, &mut vec![member_index], false)?;
10574                for mut descriptor in parallel_descriptors(children, member_base)? {
10575                    let flat_index = cursor + descriptor.offset;
10576                    descriptor.group_path.insert(
10577                        0,
10578                        selection_group_entry(
10579                            base_sequence,
10580                            total_size,
10581                            flat_index,
10582                            group_kind,
10583                            &SelectionMemberMetadata {
10584                                key: key.clone(),
10585                                index: member_index,
10586                                base_sequence: member_base,
10587                                size: member_size,
10588                                kind: member_kind.clone(),
10589                            },
10590                        ),
10591                    );
10592                    descriptor.member_path.insert(0, member_index);
10593                    descriptor.offset = flat_index;
10594                    descriptors.push(descriptor);
10595                }
10596            }
10597            operation => {
10598                validate_parallel_operations(
10599                    std::slice::from_ref(&operation),
10600                    &mut Vec::new(),
10601                    true,
10602                )?;
10603                descriptors.push(ParallelDescriptor {
10604                    operation,
10605                    offset: cursor,
10606                    member_path: vec![member_index],
10607                    group_path: vec![selection_group_entry(
10608                        base_sequence,
10609                        total_size,
10610                        cursor,
10611                        group_kind,
10612                        &SelectionMemberMetadata {
10613                            key: key.clone(),
10614                            index: member_index,
10615                            base_sequence: member_base,
10616                            size: member_size,
10617                            kind: member_kind.clone(),
10618                        },
10619                    )],
10620                });
10621            }
10622        }
10623        members.push(SelectionMemberPlan {
10624            key,
10625            index: member_index,
10626            base_sequence: member_base,
10627            size: member_size,
10628            kind: member_kind,
10629            shape: member_shape,
10630            leaf_start,
10631        });
10632        cursor += member_size;
10633    }
10634    Ok((descriptors, members))
10635}
10636
10637struct SelectionLeaf {
10638    call: ParallelLeafCall,
10639    outcome: Option<Result<ParallelAvroResult>>,
10640}
10641
10642/// Stable reference to one member of a durable selection group.
10643#[derive(Clone)]
10644pub struct DurableOperationHandle {
10645    ctx: WorkflowContext,
10646    pub key: SelectionKey,
10647    pub index: usize,
10648    pub kind: String,
10649    pub identity: String,
10650    pub base_sequence: u64,
10651    pub size: usize,
10652    pub selection_group_id: String,
10653    shape: ParallelShape,
10654}
10655
10656impl std::fmt::Debug for DurableOperationHandle {
10657    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10658        formatter
10659            .debug_struct("DurableOperationHandle")
10660            .field("key", &self.key)
10661            .field("index", &self.index)
10662            .field("kind", &self.kind)
10663            .field("identity", &self.identity)
10664            .field("base_sequence", &self.base_sequence)
10665            .field("size", &self.size)
10666            .field("selection_group_id", &self.selection_group_id)
10667            .finish()
10668    }
10669}
10670
10671impl DurableOperationHandle {
10672    /// Await this member after another member has already won the selection.
10673    pub fn await_result(&self) -> DurableOperationAwaitCall {
10674        DurableOperationAwaitCall {
10675            handle: self.clone(),
10676        }
10677    }
10678
10679    /// Request cancellation without affecting siblings. The future resolves
10680    /// to unit; only `SelectionOperationCancelled` history proves cancellation
10681    /// beat a concurrently committed terminal result.
10682    pub fn cancel(&self) -> CancelDurableOperationCall {
10683        CancelDurableOperationCall {
10684            handle: self.clone(),
10685            emitted: false,
10686        }
10687    }
10688}
10689
10690/// The one winner committed for a durable selection group.
10691#[derive(Debug)]
10692pub struct SelectionResult {
10693    pub key: SelectionKey,
10694    pub index: usize,
10695    pub kind: String,
10696    pub identity: String,
10697    pub value: Option<ParallelResult>,
10698    pub failure: Option<Error>,
10699    pub winner: DurableOperationHandle,
10700    pub handles: Vec<DurableOperationHandle>,
10701}
10702
10703impl SelectionResult {
10704    pub fn succeeded(&self) -> bool {
10705        self.failure.is_none()
10706    }
10707
10708    pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10709        self.handles.iter().find(|handle| &handle.key == key)
10710    }
10711
10712    pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10713        self.handles
10714            .iter()
10715            .filter(|handle| handle.index != self.index)
10716            .collect()
10717    }
10718
10719    pub fn into_result(self) -> Result<ParallelResult> {
10720        match (self.value, self.failure) {
10721            (Some(value), None) => Ok(value),
10722            (_, Some(error)) => Err(error),
10723            _ => Err(Error::WorkerLoop(
10724                "selection result contained neither a value nor a failure".to_string(),
10725            )),
10726        }
10727    }
10728}
10729
10730/// Future returned by [`WorkflowContext::select`].
10731pub struct SelectCall {
10732    ctx: WorkflowContext,
10733    operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10734    members: Vec<SelectionMemberPlan>,
10735    leaves: Vec<SelectionLeaf>,
10736    group_id: Option<String>,
10737}
10738
10739impl SelectCall {
10740    fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10741        Self {
10742            ctx,
10743            operations: Some(operations),
10744            members: Vec::new(),
10745            leaves: Vec::new(),
10746            group_id: None,
10747        }
10748    }
10749
10750    fn initialize(&mut self) -> Result<()> {
10751        let operations = self.operations.take().unwrap_or_default();
10752        let base_sequence = {
10753            let state = self
10754                .ctx
10755                .state
10756                .lock()
10757                .map_err(|_| Error::WorkflowStatePoisoned)?;
10758            if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10759                marker.selection_group_base_sequence
10760            } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10761                recorded.sequence()
10762            } else {
10763                let last = state
10764                    .recorded_commands
10765                    .last()
10766                    .map(RecordedCommand::sequence)
10767                    .unwrap_or(0);
10768                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10769                    .and_then(|sequence| sequence.checked_add(1))
10770                    .ok_or(Error::TimerDurationOverflow)?
10771            }
10772        };
10773        let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10774        let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10775        self.leaves = descriptors
10776            .into_iter()
10777            .map(|descriptor| SelectionLeaf {
10778                call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10779                outcome: None,
10780            })
10781            .collect();
10782        self.members = members;
10783        self.group_id = Some(group_id);
10784        Ok(())
10785    }
10786}
10787
10788impl Future for SelectCall {
10789    type Output = Result<SelectionResult>;
10790
10791    fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10792        if self.operations.is_some() {
10793            if let Err(error) = self.initialize() {
10794                return Poll::Ready(Err(error));
10795            }
10796        }
10797
10798        for leaf in &mut self.leaves {
10799            if leaf.outcome.is_some() {
10800                continue;
10801            }
10802            if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10803                if outcome
10804                    .as_ref()
10805                    .err()
10806                    .is_some_and(workflow_task_integrity_error)
10807                {
10808                    return Poll::Ready(outcome.map(|_| unreachable!()));
10809                }
10810                leaf.outcome = Some(outcome);
10811            }
10812        }
10813
10814        let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10815        let selection_member_range = self
10816            .members
10817            .first()
10818            .map(|member| member.base_sequence)
10819            .zip(self.leaves.len().try_into().ok())
10820            .map(|(base_sequence, size): (u64, u64)| {
10821                base_sequence..base_sequence.saturating_add(size)
10822            });
10823        let marker = {
10824            let mut state = match self.ctx.state.lock() {
10825                Ok(state) => state,
10826                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10827            };
10828            let marker = state
10829                .selection_markers
10830                .get(state.selection_marker_cursor)
10831                .cloned();
10832            if marker.is_none()
10833                && all_members_terminal
10834                && selection_member_range.as_ref().is_some_and(|member_range| {
10835                    state
10836                        .recorded_commands
10837                        .iter()
10838                        .any(|command| member_range.contains(&command.sequence()))
10839                })
10840            {
10841                // Terminal member history can be committed before the server's
10842                // SelectionResolved marker becomes visible to this task. That
10843                // marker is the durable barrier the selection is still waiting
10844                // on, so an otherwise commandless replay remains legitimately
10845                // pending instead of failing as an untracked yield.
10846                state.matched_recorded_pending = true;
10847            }
10848            marker
10849        };
10850        let Some(marker) = marker else {
10851            return Poll::Pending;
10852        };
10853        if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10854            || marker.selection_group_size != self.leaves.len()
10855            || self.members.first().map(|member| member.base_sequence)
10856                != Some(marker.selection_group_base_sequence)
10857        {
10858            return Poll::Ready(Err(invalid_recorded_history(
10859                "selection_group_shape_mismatch",
10860                marker.selection_group_base_sequence,
10861                self.group_id
10862                    .as_deref()
10863                    .unwrap_or("initialized selection group"),
10864                &marker.selection_group_id,
10865                "recorded selection group differs from current workflow code",
10866            )));
10867        }
10868        let Some(member_position) = self.members.iter().position(|member| {
10869            member.key == marker.member_key
10870                && member.index == marker.member_index
10871                && member.base_sequence == marker.member_base_sequence
10872                && member.size == marker.member_size
10873                && member.kind == marker.operation_kind
10874        }) else {
10875            return Poll::Ready(Err(invalid_recorded_history(
10876                "selection_member_shape_mismatch",
10877                marker.member_base_sequence,
10878                "winner member matching current workflow code",
10879                &format!("{:?}", marker.member_key),
10880                "recorded selection winner differs from the authored member identity",
10881            )));
10882        };
10883        let member = self.members[member_position].clone();
10884        let (handles, resolution_sequence) = {
10885            let mut state = match self.ctx.state.lock() {
10886                Ok(state) => state,
10887                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10888            };
10889            let identities = self
10890                .members
10891                .iter()
10892                .map(|candidate| {
10893                    selection_operation_identity(
10894                        &state,
10895                        &candidate.kind,
10896                        candidate.base_sequence,
10897                        candidate.size,
10898                    )
10899                })
10900                .collect::<Vec<_>>();
10901            if let Some((position, missing)) = identities
10902                .iter()
10903                .enumerate()
10904                .find(|(_, identity)| identity.is_empty())
10905                .map(|(position, identity)| (position, identity.clone()))
10906            {
10907                let candidate = &self.members[position];
10908                return Poll::Ready(Err(invalid_recorded_history(
10909                    "selection_operation_identity_missing",
10910                    candidate.base_sequence,
10911                    &format!(
10912                        "durable {} resource identity from scheduled/open history",
10913                        candidate.kind
10914                    ),
10915                    &missing,
10916                    "selection member history is missing its canonical durable identity",
10917                )));
10918            }
10919            let expected_winner_identity = &identities[member_position];
10920            let resolution_sequence = match validated_selection_resolution_sequence(
10921                &state,
10922                &marker,
10923                &member,
10924                expected_winner_identity,
10925            ) {
10926                Ok(sequence) => sequence,
10927                Err(error) => return Poll::Ready(Err(error)),
10928            };
10929            let handles = self
10930                .members
10931                .iter()
10932                .zip(identities)
10933                .map(|(member, identity)| DurableOperationHandle {
10934                    ctx: self.ctx.clone(),
10935                    key: member.key.clone(),
10936                    index: member.index,
10937                    kind: member.kind.clone(),
10938                    identity,
10939                    base_sequence: member.base_sequence,
10940                    size: member.size,
10941                    selection_group_id: marker.selection_group_id.clone(),
10942                    shape: member.shape.clone(),
10943                })
10944                .collect::<Vec<_>>();
10945            if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
10946                return Poll::Ready(Err(error));
10947            }
10948            state.selection_marker_cursor += 1;
10949            (handles, resolution_sequence)
10950        };
10951
10952        let mut winner_failure = None;
10953        let mut flat_results = Vec::with_capacity(member.size);
10954        if marker.outcome == "failed" {
10955            let resolution_offset = match resolution_sequence
10956                .checked_sub(member.base_sequence)
10957                .and_then(|offset| usize::try_from(offset).ok())
10958            {
10959                Some(offset) if offset < member.size => offset,
10960                _ => {
10961                    return Poll::Ready(Err(invalid_recorded_history(
10962                        "selection_resolution_event_mismatch",
10963                        member.base_sequence,
10964                        "failure event within selected member bounds",
10965                        &resolution_sequence.to_string(),
10966                        "selection failure event is outside the authored member",
10967                    )))
10968                }
10969            };
10970            let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
10971            match leaf.outcome.take() {
10972                Some(Err(error)) => winner_failure = Some(error),
10973                _ => {
10974                    return Poll::Ready(Err(invalid_recorded_history(
10975                        "selection_winner_outcome_mismatch",
10976                        member.base_sequence,
10977                        "exact failed terminal history referenced by SelectionResolved",
10978                        "missing or successful resolution event",
10979                        "selection winner marker disagrees with terminal operation history",
10980                    )))
10981                }
10982            }
10983        } else {
10984            for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
10985                match leaf.outcome.take() {
10986                    Some(Ok(result)) => flat_results.push(result),
10987                    Some(Err(_)) => {
10988                        return Poll::Ready(Err(invalid_recorded_history(
10989                            "selection_winner_outcome_mismatch",
10990                            member.base_sequence,
10991                            "fully completed nested selection member",
10992                            "failed durable leaf",
10993                            "completed selection winner contains a failed leaf",
10994                        )))
10995                    }
10996                    None => {
10997                        return Poll::Ready(Err(invalid_recorded_history(
10998                            "selection_winner_unresolved",
10999                            member.base_sequence,
11000                            "terminal history for every completed winner leaf",
11001                            "pending member history",
11002                            "completed SelectionResolved member has an unfinished durable barrier",
11003                        )))
11004                    }
11005                }
11006            }
11007        }
11008        let value = if winner_failure.is_none() {
11009            let mut flat_results = flat_results.into_iter();
11010            let value = parallel_results_for_shape(&member.shape, &mut flat_results);
11011            match value.into_json_result() {
11012                Ok(value) => Some(value),
11013                Err(error) => return Poll::Ready(Err(error)),
11014            }
11015        } else {
11016            None
11017        };
11018        let winner = handles[member_position].clone();
11019        Poll::Ready(Ok(SelectionResult {
11020            key: winner.key.clone(),
11021            index: winner.index,
11022            kind: winner.kind.clone(),
11023            identity: winner.identity.clone(),
11024            value,
11025            failure: winner_failure,
11026            winner,
11027            handles,
11028        }))
11029    }
11030}
11031
11032fn selection_operation_identity(
11033    state: &WorkflowState,
11034    kind: &str,
11035    base_sequence: u64,
11036    size: usize,
11037) -> String {
11038    if kind == "group" {
11039        return format!("group:{base_sequence}:{size}");
11040    }
11041    let fields: &[&str] = match kind {
11042        "activity" => &["activity_execution_id"],
11043        "child" => &["child_workflow_run_id"],
11044        "timer" => &["timer_id"],
11045        "signal" => &["signal_wait_id"],
11046        "condition" => &["condition_wait_id"],
11047        _ => &[],
11048    };
11049    for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11050        for event in state
11051            .history_events
11052            .iter()
11053            .filter(|event| durable_event_sequence(event) == Some(sequence))
11054        {
11055            for field in fields {
11056                if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11057                    if !identity.is_empty() {
11058                        return identity.to_string();
11059                    }
11060                }
11061            }
11062        }
11063    }
11064    String::new()
11065}
11066
11067fn validated_selection_resolution_sequence(
11068    state: &WorkflowState,
11069    marker: &SelectionMarker,
11070    member: &SelectionMemberPlan,
11071    expected_identity: &str,
11072) -> Result<u64> {
11073    if expected_identity.is_empty() {
11074        return Err(invalid_recorded_history(
11075            "selection_operation_identity_missing",
11076            member.base_sequence,
11077            &format!(
11078                "durable {} resource identity from scheduled/open history",
11079                member.kind
11080            ),
11081            "missing operation identity",
11082            "selection member history is missing its canonical durable identity",
11083        ));
11084    }
11085    if marker.operation_identity != expected_identity {
11086        return Err(invalid_recorded_history(
11087            "selection_operation_identity_mismatch",
11088            member.base_sequence,
11089            expected_identity,
11090            &marker.operation_identity,
11091            "selection winner identity does not match durable scheduled/open history",
11092        ));
11093    }
11094
11095    let failure_types = [
11096        "ActivityFailed",
11097        "ActivityCancelled",
11098        "ActivityTimedOut",
11099        "ChildRunFailed",
11100        "ChildRunCancelled",
11101        "ChildRunTerminated",
11102    ];
11103    let success_types = [
11104        "ActivityCompleted",
11105        "ChildRunCompleted",
11106        "TimerFired",
11107        "SignalApplied",
11108        "ConditionWaitSatisfied",
11109        "ConditionWaitTimedOut",
11110    ];
11111    let terminal_types: &[&str] = if marker.outcome == "failed" {
11112        &failure_types
11113    } else {
11114        &success_types
11115    };
11116    let mut candidates = Vec::new();
11117    for event in state.history_events.iter() {
11118        let Some(sequence) = durable_event_sequence(event) else {
11119            continue;
11120        };
11121        if sequence < member.base_sequence
11122            || sequence >= member.base_sequence.saturating_add(member.size as u64)
11123            || !terminal_types.contains(&event.event_type.as_str())
11124        {
11125            continue;
11126        }
11127        let event_id = event
11128            .raw
11129            .get("id")
11130            .or_else(|| event.raw.get("event_id"))
11131            .and_then(Value::as_str)
11132            .filter(|value| !value.is_empty())
11133            .ok_or_else(|| {
11134                invalid_recorded_history(
11135                    "selection_resolution_event_id_missing",
11136                    member.base_sequence,
11137                    "terminal selection history with a durable event id",
11138                    &event.payload.to_string(),
11139                    "selection terminal history cannot be bound to its winner marker",
11140                )
11141            })?;
11142        candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11143    }
11144    let resolution = if marker.outcome == "failed" {
11145        candidates.first()
11146    } else {
11147        candidates.last()
11148    };
11149    let Some((event_id, event_type, sequence)) = resolution else {
11150        return Err(invalid_recorded_history(
11151            "selection_resolution_event_missing",
11152            member.base_sequence,
11153            "terminal history for the selected member",
11154            &format!("{:?}", marker.member_key),
11155            "selection winner marker has no matching durable terminal event",
11156        ));
11157    };
11158    if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11159        return Err(invalid_recorded_history(
11160            "selection_resolution_event_mismatch",
11161            member.base_sequence,
11162            &format!("{event_type}:{event_id}"),
11163            &format!(
11164                "{}:{}",
11165                marker.resolution_event_type, marker.resolution_event_id
11166            ),
11167            "selection winner marker does not reference the event that made its member terminal",
11168        ));
11169    }
11170    Ok(*sequence)
11171}
11172
11173fn recorded_selection_member_outcome(
11174    state: &WorkflowState,
11175    handle: &DurableOperationHandle,
11176) -> Result<Option<ParallelResult>> {
11177    for event in state.history_events.iter() {
11178        let Some(sequence) = durable_event_sequence(event) else {
11179            continue;
11180        };
11181        if sequence < handle.base_sequence
11182            || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11183            || !matches!(
11184                event.event_type.as_str(),
11185                "ActivityFailed"
11186                    | "ActivityCancelled"
11187                    | "ActivityTimedOut"
11188                    | "ChildRunFailed"
11189                    | "ChildRunCancelled"
11190                    | "ChildRunTerminated"
11191            )
11192        {
11193            continue;
11194        }
11195        let Some(command) = state
11196            .recorded_commands
11197            .iter()
11198            .find(|command| command.sequence() == sequence)
11199        else {
11200            continue;
11201        };
11202        match command {
11203            RecordedCommand::Activity {
11204                outcome: Some(Err(failure)),
11205                ..
11206            } => return Err(Error::ActivityFailed(failure.clone())),
11207            RecordedCommand::ChildWorkflow {
11208                outcome: Some(Err(failure)),
11209                ..
11210            } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11211            _ => {}
11212        }
11213    }
11214
11215    let mut results = Vec::with_capacity(handle.size);
11216    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11217        let Some(command) = state
11218            .recorded_commands
11219            .iter()
11220            .find(|command| command.sequence() == sequence)
11221        else {
11222            return Ok(None);
11223        };
11224        let result = match command {
11225            RecordedCommand::Activity { outcome, .. } => match outcome {
11226                Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11227                Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11228                None => return Ok(None),
11229            },
11230            RecordedCommand::Timer { fired, .. } => {
11231                if !fired {
11232                    return Ok(None);
11233                }
11234                ParallelAvroResult::Timer
11235            }
11236            RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11237                Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11238                Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11239                None => return Ok(None),
11240            },
11241            RecordedCommand::SignalWait { value, .. } => match value {
11242                Some(value) => ParallelAvroResult::Signal(value.clone()),
11243                None => return Ok(None),
11244            },
11245            RecordedCommand::ConditionWait { result, .. } => match result {
11246                Some(result) => ParallelAvroResult::Condition(*result),
11247                None => return Ok(None),
11248            },
11249            other => {
11250                return Err(command_mismatch(
11251                    other,
11252                    format!("selected {} member", handle.kind),
11253                ))
11254            }
11255        };
11256        results.push(result);
11257    }
11258    let mut results = results.into_iter();
11259    parallel_results_for_shape(&handle.shape, &mut results)
11260        .into_json_result()
11261        .map(Some)
11262}
11263
11264fn recorded_selection_member_is_terminal(
11265    state: &WorkflowState,
11266    handle: &DurableOperationHandle,
11267) -> bool {
11268    let mut completed = 0usize;
11269    let mut all_completed = true;
11270    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11271        let Some(command) = state
11272            .recorded_commands
11273            .iter()
11274            .find(|command| command.sequence() == sequence)
11275        else {
11276            all_completed = false;
11277            continue;
11278        };
11279        let terminal = match command {
11280            RecordedCommand::Activity {
11281                outcome: Some(Err(_)),
11282                ..
11283            }
11284            | RecordedCommand::ChildWorkflow {
11285                outcome: Some(Err(_)),
11286                ..
11287            } => return true,
11288            RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11289            RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11290            RecordedCommand::Timer { fired, .. } => *fired,
11291            RecordedCommand::SignalWait { value, .. } => value.is_some(),
11292            RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11293            RecordedCommand::SearchAttributes { .. }
11294            | RecordedCommand::SideEffect { .. }
11295            | RecordedCommand::VersionMarker { .. }
11296            | RecordedCommand::Memo { .. } => false,
11297        };
11298        if !terminal {
11299            all_completed = false;
11300            continue;
11301        }
11302        completed += 1;
11303    }
11304    all_completed && completed == handle.size
11305}
11306
11307fn selection_cancellation_for_handle(
11308    state: &WorkflowState,
11309    handle: &DurableOperationHandle,
11310) -> Result<bool> {
11311    let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11312        recorded.selection_group_id == handle.selection_group_id
11313            && recorded.member_base_sequence == handle.base_sequence
11314    }) else {
11315        return Ok(false);
11316    };
11317    validate_selection_cancellation_marker(marker, handle)?;
11318    Ok(true)
11319}
11320
11321fn validate_selection_cancellations_for_handles(
11322    state: &WorkflowState,
11323    handles: &[DurableOperationHandle],
11324) -> Result<()> {
11325    let Some(group_id) = handles
11326        .first()
11327        .map(|handle| handle.selection_group_id.as_str())
11328    else {
11329        return Ok(());
11330    };
11331    for marker in state
11332        .cancelled_selection_members
11333        .iter()
11334        .filter(|marker| marker.selection_group_id == group_id)
11335    {
11336        let Some(handle) = handles
11337            .iter()
11338            .find(|handle| handle.base_sequence == marker.member_base_sequence)
11339        else {
11340            return Err(invalid_recorded_history(
11341                "selection_cancellation_member_mismatch",
11342                marker.member_base_sequence,
11343                "SelectionOperationCancelled matching an authored selection handle",
11344                &format!("{marker:?}"),
11345                "selection cancellation member base does not name an authored member",
11346            ));
11347        };
11348        validate_selection_cancellation_marker(marker, handle)?;
11349    }
11350    Ok(())
11351}
11352
11353fn validate_selection_cancellation_marker(
11354    marker: &SelectionCancellation,
11355    handle: &DurableOperationHandle,
11356) -> Result<()> {
11357    if marker.selection_group_id != handle.selection_group_id
11358        || marker.member_key != handle.key
11359        || marker.member_index != handle.index
11360        || marker.member_base_sequence != handle.base_sequence
11361        || marker.member_size != handle.size
11362        || marker.operation_kind != handle.kind
11363        || marker.operation_identity != handle.identity
11364    {
11365        return Err(invalid_recorded_history(
11366            "selection_cancellation_member_mismatch",
11367            handle.base_sequence,
11368            "SelectionOperationCancelled matching the authored selection handle",
11369            &format!("{marker:?}"),
11370            "selection cancellation history targets different authored member metadata",
11371        ));
11372    }
11373    Ok(())
11374}
11375
11376/// Future returned by [`DurableOperationHandle::await_result`].
11377pub struct DurableOperationAwaitCall {
11378    handle: DurableOperationHandle,
11379}
11380
11381impl Future for DurableOperationAwaitCall {
11382    type Output = Result<ParallelResult>;
11383
11384    fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11385        let state = match self.handle.ctx.state.lock() {
11386            Ok(state) => state,
11387            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11388        };
11389        match selection_cancellation_for_handle(&state, &self.handle) {
11390            Err(error) => return Poll::Ready(Err(error)),
11391            Ok(false) => {}
11392            Ok(true) => {
11393                return Poll::Ready(Err(Error::DurableOperationCancelled(
11394                    DurableOperationCancelled {
11395                        selection_group_id: self.handle.selection_group_id.clone(),
11396                        member_key: self.handle.key.clone(),
11397                        member_index: self.handle.index,
11398                        operation_kind: self.handle.kind.clone(),
11399                        operation_identity: self.handle.identity.clone(),
11400                    },
11401                )));
11402            }
11403        }
11404        match recorded_selection_member_outcome(&state, &self.handle) {
11405            Ok(Some(result)) => Poll::Ready(Ok(result)),
11406            Ok(None) => Poll::Pending,
11407            Err(error) => Poll::Ready(Err(error)),
11408        }
11409    }
11410}
11411
11412/// Future returned by [`DurableOperationHandle::cancel`].
11413pub struct CancelDurableOperationCall {
11414    handle: DurableOperationHandle,
11415    emitted: bool,
11416}
11417
11418impl Future for CancelDurableOperationCall {
11419    type Output = Result<()>;
11420
11421    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11422        let ctx = self.handle.ctx.clone();
11423        let mut state = match ctx.state.lock() {
11424            Ok(state) => state,
11425            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11426        };
11427        match selection_cancellation_for_handle(&state, &self.handle) {
11428            Err(error) => return Poll::Ready(Err(error)),
11429            Ok(true) => return Poll::Ready(Ok(())),
11430            Ok(false) => {}
11431        }
11432        if recorded_selection_member_is_terminal(&state, &self.handle) {
11433            return Poll::Ready(Ok(()));
11434        }
11435        if !self.emitted {
11436            state.commands.push(json!({
11437                "type": "cancel_selection_operation",
11438                "selection_group_id": self.handle.selection_group_id,
11439                "member_key": self.handle.key,
11440                "member_index": self.handle.index,
11441                "member_base_sequence": self.handle.base_sequence,
11442                "member_size": self.handle.size,
11443                "operation_kind": self.handle.kind,
11444                "operation_identity": self.handle.identity,
11445            }));
11446            self.emitted = true;
11447        }
11448        // The cancellation command is only a request. Do not expose workflow
11449        // state after cancel until committed SelectionOperationCancelled
11450        // history proves that the request won the race with member completion.
11451        Poll::Pending
11452    }
11453}
11454
11455trait PollNestedResultExt<T> {
11456    fn flatten_result(self) -> Poll<Result<T>>;
11457}
11458
11459impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11460    fn flatten_result(self) -> Poll<Result<T>> {
11461        match self {
11462            Poll::Ready(Ok(result)) => Poll::Ready(result),
11463            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11464            Poll::Pending => Poll::Pending,
11465        }
11466    }
11467}
11468
11469struct SagaCompensation {
11470    activity_type: String,
11471    options: ActivityOptions,
11472    arguments: AvroValue,
11473    registration_order: usize,
11474}
11475
11476/// Workflow-local deterministic saga compensation helper.
11477///
11478/// Register each compensation only after its forward step succeeds. Passing
11479/// the forward `Result` to [`Saga::finish`] runs compensations sequentially in
11480/// reverse registration order after any failure, including cooperative
11481/// cancellation. Each compensation is an ordinary durable activity, so replay,
11482/// duplicate delivery, and worker restart use existing history semantics.
11483pub struct Saga {
11484    ctx: WorkflowContext,
11485    compensations: Vec<SagaCompensation>,
11486}
11487
11488impl Saga {
11489    fn new(ctx: WorkflowContext) -> Self {
11490        Self {
11491            ctx,
11492            compensations: Vec::new(),
11493        }
11494    }
11495
11496    pub fn add_compensation<T: Serialize>(
11497        &mut self,
11498        activity_type: impl Into<String>,
11499        args: T,
11500    ) -> Result<&mut Self> {
11501        self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11502    }
11503
11504    pub fn add_compensation_with_options<T: Serialize>(
11505        &mut self,
11506        activity_type: impl Into<String>,
11507        options: ActivityOptions,
11508        args: T,
11509    ) -> Result<&mut Self> {
11510        let activity_type = activity_type.into();
11511        if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11512            return Err(Error::Codec(
11513                "saga compensation activity type must be non-empty without surrounding whitespace"
11514                    .to_string(),
11515            ));
11516        }
11517        options.validate().map_err(Error::InvalidActivityOptions)?;
11518        let arguments = AvroValue::from_serialize(&args)?;
11519        let registration_order = self.compensations.len() + 1;
11520        self.compensations.push(SagaCompensation {
11521            activity_type,
11522            options,
11523            arguments,
11524            registration_order,
11525        });
11526        Ok(self)
11527    }
11528
11529    /// Compensate `initiating_failure` and return the failure that must remain.
11530    pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11531        while let Some(compensation) = self.compensations.pop() {
11532            if let Err(compensation_failure) = self
11533                .ctx
11534                .activity_with_options(
11535                    compensation.activity_type.clone(),
11536                    compensation.options,
11537                    compensation.arguments,
11538                )
11539                .await
11540            {
11541                if workflow_task_integrity_error(&compensation_failure) {
11542                    return compensation_failure;
11543                }
11544                return Error::SagaCompensationFailed(SagaCompensationFailure {
11545                    initiating_failure: Box::new(initiating_failure),
11546                    compensation_failure: Box::new(compensation_failure),
11547                    compensation_activity_type: compensation.activity_type,
11548                    compensation_registration_order: compensation.registration_order,
11549                });
11550            }
11551        }
11552        initiating_failure
11553    }
11554
11555    /// Return a successful forward value or compensate and preserve its failure.
11556    pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11557        match outcome {
11558            Ok(value) => Ok(value),
11559            Err(error) => Err(self.compensate(error).await),
11560        }
11561    }
11562}
11563
11564pub struct ActivityCall {
11565    ctx: WorkflowContext,
11566    activity_type: String,
11567    options: ActivityOptions,
11568    args: Option<Result<AvroValue>>,
11569    scheduled: bool,
11570    parallel_group_path: Vec<ParallelGroupMetadata>,
11571}
11572
11573impl ActivityCall {
11574    fn poll_avro_value(
11575        mut self: Pin<&mut Self>,
11576        _cx: &mut TaskContext<'_>,
11577    ) -> Poll<Result<AvroValue>> {
11578        let ctx = self.ctx.clone();
11579        let mut state = match ctx.state.lock() {
11580            Ok(state) => state,
11581            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11582        };
11583
11584        if self.scheduled {
11585            return Poll::Pending;
11586        }
11587
11588        let options = match self.options.validate() {
11589            Ok(options) => options,
11590            Err(error) => {
11591                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11592            }
11593        };
11594        let task_queue = options
11595            .task_queue
11596            .clone()
11597            .unwrap_or_else(|| state.task_queue.clone());
11598        let current_recorded_options = RecordedActivityOptions {
11599            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11600            // Rust schedules ordinary durable activities. The server records a
11601            // non-null mode only for a specialized execution primitive.
11602            execution_mode: RecordedSnapshotValue::Known(None),
11603            retry_policy: current_activity_retry_snapshot(&options),
11604        };
11605
11606        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11607            let sequence = recorded.sequence();
11608            match recorded {
11609                RecordedCommand::Activity {
11610                    activity_type,
11611                    options: recorded_options,
11612                    outcome,
11613                    parallel_group_path,
11614                    ..
11615                } => {
11616                    if let Err(error) = ensure_parallel_path_matches(
11617                        sequence,
11618                        parallel_group_path.as_deref(),
11619                        &self.parallel_group_path,
11620                    ) {
11621                        return Poll::Ready(Err(error));
11622                    }
11623                    if let Some(recorded_type) = activity_type {
11624                        if recorded_type != self.activity_type {
11625                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11626                                ReplayFailure::new(
11627                                    "recorded_command_detail_mismatch",
11628                                    Some(sequence),
11629                                    Some(format!("activity:{recorded_type}")),
11630                                    Some(format!("activity:{}", self.activity_type)),
11631                                    "recorded activity type differs from the current workflow command",
11632                                ),
11633                            )));
11634                        }
11635                    }
11636                    if let Some(recorded_options) = recorded_options {
11637                        if !recorded_options
11638                            .task_queue
11639                            .matches_current(&current_recorded_options.task_queue)
11640                        {
11641                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11642                                ReplayFailure::new(
11643                                    "activity_task_queue_mismatch",
11644                                    Some(sequence),
11645                                    Some(activity_options_description(&recorded_options)),
11646                                    Some(activity_options_description(&current_recorded_options)),
11647                                    "recorded activity task queue differs from the current workflow command",
11648                                ),
11649                            )));
11650                        }
11651                        if !recorded_options
11652                            .execution_mode
11653                            .matches_current(&current_recorded_options.execution_mode)
11654                        {
11655                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11656                                ReplayFailure::new(
11657                                    "activity_execution_mode_mismatch",
11658                                    Some(sequence),
11659                                    Some(activity_options_description(&recorded_options)),
11660                                    Some(activity_options_description(&current_recorded_options)),
11661                                    "recorded activity execution mode differs from the current workflow command",
11662                                ),
11663                            )));
11664                        }
11665                        if !recorded_options
11666                            .retry_policy
11667                            .matches_current(&current_recorded_options.retry_policy)
11668                        {
11669                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11670                                ReplayFailure::new(
11671                                    "activity_retry_policy_mismatch",
11672                                    Some(sequence),
11673                                    Some(activity_options_description(&recorded_options)),
11674                                    Some(activity_options_description(&current_recorded_options)),
11675                                    "recorded activity retry policy differs from the current workflow command",
11676                                ),
11677                            )));
11678                        }
11679                    }
11680                    state.command_cursor += 1;
11681                    if let Some(outcome) = outcome {
11682                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11683                    }
11684                    state.matched_recorded_pending = true;
11685                    self.scheduled = true;
11686                    return Poll::Pending;
11687                }
11688                other => {
11689                    return Poll::Ready(Err(command_mismatch(
11690                        &other,
11691                        format!("activity:{}", self.activity_type),
11692                    )));
11693                }
11694            }
11695        }
11696
11697        if !self.scheduled {
11698            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11699                Ok(args) => args,
11700                Err(error) => return Poll::Ready(Err(error)),
11701            };
11702            let arguments = normalize_avro_arguments(args);
11703            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11704                Ok(envelope) => envelope,
11705                Err(error) => return Poll::Ready(Err(error)),
11706            };
11707
11708            let mut command = serde_json::Map::from_iter([
11709                ("type".to_string(), json!("schedule_activity")),
11710                (
11711                    "activity_type".to_string(),
11712                    json!(self.activity_type.clone()),
11713                ),
11714                ("queue".to_string(), json!(task_queue)),
11715                ("arguments".to_string(), envelope),
11716            ]);
11717            for (field, value) in [
11718                ("start_to_close_timeout", options.start_to_close_timeout),
11719                (
11720                    "schedule_to_start_timeout",
11721                    options.schedule_to_start_timeout,
11722                ),
11723                (
11724                    "schedule_to_close_timeout",
11725                    options.schedule_to_close_timeout,
11726                ),
11727                ("heartbeat_timeout", options.heartbeat_timeout),
11728            ] {
11729                if let Some(value) = value {
11730                    command.insert(field.to_string(), json!(value));
11731                }
11732            }
11733            if let Some(retry_policy) = options.retry_policy {
11734                command.insert("retry_policy".to_string(), retry_policy);
11735            }
11736            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11737            state.commands.push(Value::Object(command));
11738            self.scheduled = true;
11739        }
11740
11741        Poll::Pending
11742    }
11743}
11744
11745impl Future for ActivityCall {
11746    type Output = Result<Value>;
11747
11748    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11749        match self.poll_avro_value(cx) {
11750            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11751            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11752            Poll::Pending => Poll::Pending,
11753        }
11754    }
11755}
11756
11757/// Future returned by [`WorkflowContext::sleep`].
11758pub struct TimerCall {
11759    ctx: WorkflowContext,
11760    delay_seconds: Option<u64>,
11761    scheduled: bool,
11762    matched_pending: bool,
11763    parallel_group_path: Vec<ParallelGroupMetadata>,
11764}
11765
11766impl Future for TimerCall {
11767    type Output = Result<()>;
11768
11769    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11770        if self.matched_pending {
11771            return Poll::Pending;
11772        }
11773
11774        let ctx = self.ctx.clone();
11775        let Some(requested_delay) = self.delay_seconds else {
11776            return Poll::Ready(Err(Error::TimerDurationOverflow));
11777        };
11778        let mut state = match ctx.state.lock() {
11779            Ok(state) => state,
11780            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11781        };
11782
11783        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11784            match recorded {
11785                RecordedCommand::Timer {
11786                    sequence,
11787                    delay_seconds,
11788                    fired,
11789                    parallel_group_path,
11790                    ..
11791                } => {
11792                    if let Err(error) = ensure_parallel_path_matches(
11793                        sequence,
11794                        parallel_group_path.as_deref(),
11795                        &self.parallel_group_path,
11796                    ) {
11797                        return Poll::Ready(Err(error));
11798                    }
11799                    if delay_seconds != requested_delay {
11800                        return Poll::Ready(Err(Error::NonDeterministicReplay(
11801                            ReplayFailure::new(
11802                                "timer_delay_mismatch",
11803                                Some(sequence),
11804                                Some(format!("timer:{delay_seconds}s")),
11805                                Some(format!("timer:{requested_delay}s")),
11806                                "recorded timer delay differs from the current workflow command",
11807                            ),
11808                        )));
11809                    }
11810                    state.command_cursor += 1;
11811                    if fired {
11812                        return Poll::Ready(Ok(()));
11813                    }
11814                    state.matched_recorded_pending = true;
11815                    self.scheduled = true;
11816                    self.matched_pending = true;
11817                    return Poll::Pending;
11818                }
11819                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11820            }
11821        }
11822
11823        if !self.scheduled {
11824            let mut command = serde_json::Map::from_iter([
11825                ("type".to_string(), json!("start_timer")),
11826                ("delay_seconds".to_string(), json!(requested_delay)),
11827            ]);
11828            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11829            state.commands.push(Value::Object(command));
11830            self.scheduled = true;
11831        }
11832
11833        Poll::Pending
11834    }
11835}
11836
11837/// Future returned by [`WorkflowContext::wait_condition`].
11838pub struct ConditionWaitCall {
11839    ctx: WorkflowContext,
11840    options: ConditionWaitOptions,
11841    predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11842    occurrence_id: Option<String>,
11843    opened_wait: bool,
11844    parallel_group_path: Vec<ParallelGroupMetadata>,
11845}
11846
11847impl Future for ConditionWaitCall {
11848    type Output = Result<ConditionWaitResult>;
11849
11850    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11851        if self.opened_wait {
11852            return Poll::Pending;
11853        }
11854
11855        let options = match self.options.validate() {
11856            Ok(options) => options,
11857            Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11858        };
11859        let ctx = self.ctx.clone();
11860        let occurrence_id = match self.occurrence_id.as_ref() {
11861            Some(occurrence_id) => occurrence_id.clone(),
11862            None => {
11863                let mut state = match ctx.state.lock() {
11864                    Ok(state) => state,
11865                    Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11866                };
11867                let ordinal = state.condition_wait_occurrence_counter;
11868                state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11869                    Some(next) => next,
11870                    None => {
11871                        return Poll::Ready(Err(Error::WorkerLoop(
11872                            "condition wait occurrence counter overflowed".to_string(),
11873                        )))
11874                    }
11875                };
11876                let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11877                drop(state);
11878                self.occurrence_id = Some(occurrence_id.clone());
11879                occurrence_id
11880            }
11881        };
11882
11883        let recorded_result = {
11884            let mut state = match ctx.state.lock() {
11885                Ok(state) => state,
11886                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11887            };
11888            let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11889                drop(state);
11890                return self.poll_new_condition(options);
11891            };
11892            if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11893                return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11894            }
11895
11896            let mut cursor = state.command_cursor;
11897            let mut result = None;
11898            loop {
11899                let Some(RecordedCommand::ConditionWait {
11900                    sequence,
11901                    occurrence_id: recorded_occurrence_id,
11902                    condition_key,
11903                    predicate_identity,
11904                    timeout_seconds,
11905                    result: recorded_result,
11906                    parallel_group_path,
11907                    ..
11908                }) = state.recorded_commands.get(cursor)
11909                else {
11910                    break;
11911                };
11912
11913                if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
11914                    break;
11915                }
11916                if let Err(error) = ensure_parallel_path_matches(
11917                    *sequence,
11918                    parallel_group_path.as_deref(),
11919                    &self.parallel_group_path,
11920                ) {
11921                    return Poll::Ready(Err(error));
11922                }
11923                if let Err(error) = validate_recorded_condition_wait(
11924                    *sequence,
11925                    recorded_occurrence_id,
11926                    condition_key.as_deref(),
11927                    predicate_identity,
11928                    *timeout_seconds,
11929                    &occurrence_id,
11930                    &options,
11931                ) {
11932                    return Poll::Ready(Err(error));
11933                }
11934                if result == Some(ConditionWaitResult::TimedOut) {
11935                    return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
11936                        "condition_wait_reopened_after_timeout",
11937                        Some(*sequence),
11938                        Some("timed-out condition is terminal".to_string()),
11939                        Some("another physical wait-open".to_string()),
11940                        "condition history reopened one logical wait after its durable timeout",
11941                    ))));
11942                }
11943                result = *recorded_result;
11944                cursor += 1;
11945            }
11946            state.command_cursor = cursor;
11947            result
11948        };
11949
11950        if let Some(result) = recorded_result {
11951            return Poll::Ready(Ok(result));
11952        }
11953
11954        self.poll_open_condition(options)
11955    }
11956}
11957
11958impl ConditionWaitCall {
11959    fn poll_new_condition(
11960        self: Pin<&mut Self>,
11961        options: ValidatedConditionWaitOptions,
11962    ) -> Poll<Result<ConditionWaitResult>> {
11963        self.poll_open_condition(options)
11964    }
11965
11966    fn poll_open_condition(
11967        mut self: Pin<&mut Self>,
11968        options: ValidatedConditionWaitOptions,
11969    ) -> Poll<Result<ConditionWaitResult>> {
11970        let selection_member = self
11971            .parallel_group_path
11972            .first()
11973            .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
11974        match (self.predicate)() {
11975            Ok(true) if !selection_member => {
11976                return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
11977            }
11978            Ok(_) => {}
11979            Err(error) => return Poll::Ready(Err(error)),
11980        }
11981        if options.timeout_seconds == Some(0) && !selection_member {
11982            return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
11983        }
11984
11985        let ctx = self.ctx.clone();
11986        let mut state = match ctx.state.lock() {
11987            Ok(state) => state,
11988            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11989        };
11990        let mut command = serde_json::Map::from_iter([
11991            ("type".to_string(), json!("open_condition_wait")),
11992            (
11993                "condition_wait_occurrence_id".to_string(),
11994                json!(self.occurrence_id.as_deref().unwrap_or_default()),
11995            ),
11996            ("condition_key".to_string(), json!(options.condition_key)),
11997            (
11998                "condition_definition_fingerprint".to_string(),
11999                json!(options.predicate_identity),
12000            ),
12001        ]);
12002        if let Some(timeout_seconds) = options.timeout_seconds {
12003            command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
12004        }
12005        apply_parallel_group_path(&mut command, &self.parallel_group_path);
12006        state.commands.push(Value::Object(command));
12007        drop(state);
12008        self.opened_wait = true;
12009        Poll::Pending
12010    }
12011}
12012
12013fn validate_recorded_condition_wait(
12014    sequence: u64,
12015    recorded_occurrence_id: &str,
12016    recorded_key: Option<&str>,
12017    recorded_predicate_identity: &str,
12018    recorded_timeout_seconds: Option<u64>,
12019    current_occurrence_id: &str,
12020    current: &ValidatedConditionWaitOptions,
12021) -> Result<()> {
12022    if recorded_occurrence_id != current_occurrence_id {
12023        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12024            "condition_wait_occurrence_mismatch",
12025            Some(sequence),
12026            Some(recorded_occurrence_id.to_string()),
12027            Some(current_occurrence_id.to_string()),
12028            "recorded condition occurrence differs from the current authored wait position",
12029        )));
12030    }
12031    if recorded_key != Some(current.condition_key.as_str()) {
12032        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12033            "condition_wait_key_mismatch",
12034            Some(sequence),
12035            recorded_key.map(str::to_string),
12036            Some(current.condition_key.clone()),
12037            "recorded condition identity differs from the current workflow wait",
12038        )));
12039    }
12040    if recorded_predicate_identity != current.predicate_identity {
12041        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12042            "condition_wait_predicate_mismatch",
12043            Some(sequence),
12044            Some(recorded_predicate_identity.to_string()),
12045            Some(current.predicate_identity.clone()),
12046            "recorded condition predicate behavior differs from current workflow code",
12047        )));
12048    }
12049    if recorded_timeout_seconds != current.timeout_seconds {
12050        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12051            "condition_wait_timeout_mismatch",
12052            Some(sequence),
12053            recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12054            current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12055            "recorded condition timeout differs from the current workflow wait",
12056        )));
12057    }
12058    Ok(())
12059}
12060
12061/// Future returned by [`WorkflowContext::start_child_workflow`].
12062pub struct ChildWorkflowCall {
12063    ctx: WorkflowContext,
12064    workflow_type: String,
12065    options: ChildWorkflowOptions,
12066    args: Option<Result<AvroValue>>,
12067    scheduled: bool,
12068    matched_pending: bool,
12069    parallel_group_path: Vec<ParallelGroupMetadata>,
12070}
12071
12072impl ChildWorkflowCall {
12073    fn poll_avro_value(
12074        mut self: Pin<&mut Self>,
12075        _cx: &mut TaskContext<'_>,
12076    ) -> Poll<Result<ChildWorkflowAvroResult>> {
12077        if self.matched_pending {
12078            return Poll::Pending;
12079        }
12080
12081        let ctx = self.ctx.clone();
12082        let mut state = match ctx.state.lock() {
12083            Ok(state) => state,
12084            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12085        };
12086
12087        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12088            let sequence = recorded.sequence();
12089            match recorded {
12090                RecordedCommand::ChildWorkflow {
12091                    workflow_type,
12092                    outcome,
12093                    parallel_group_path,
12094                    ..
12095                } => {
12096                    if let Err(error) = ensure_parallel_path_matches(
12097                        sequence,
12098                        parallel_group_path.as_deref(),
12099                        &self.parallel_group_path,
12100                    ) {
12101                        return Poll::Ready(Err(error));
12102                    }
12103                    if let Some(recorded_type) = workflow_type {
12104                        if recorded_type != self.workflow_type {
12105                            return Poll::Ready(Err(Error::NonDeterministicReplay(
12106                                ReplayFailure::new(
12107                                    "recorded_command_detail_mismatch",
12108                                    Some(sequence),
12109                                    Some(format!("child workflow:{recorded_type}")),
12110                                    Some(format!("child workflow:{}", self.workflow_type)),
12111                                    "recorded child workflow type differs from the current workflow command",
12112                                ),
12113                            )));
12114                        }
12115                    }
12116                    state.command_cursor += 1;
12117                    if let Some(outcome) = outcome {
12118                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12119                    }
12120                    state.matched_recorded_pending = true;
12121                    self.scheduled = true;
12122                    self.matched_pending = true;
12123                    return Poll::Pending;
12124                }
12125                other => {
12126                    return Poll::Ready(Err(command_mismatch(
12127                        &other,
12128                        format!("child workflow:{}", self.workflow_type),
12129                    )));
12130                }
12131            }
12132        }
12133
12134        if !self.scheduled {
12135            if self.options.task_queue.trim().is_empty() {
12136                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12137                    "task_queue must not be empty".to_string(),
12138                )));
12139            }
12140            for (name, value) in [
12141                (
12142                    "execution_timeout_seconds",
12143                    self.options.execution_timeout_seconds,
12144                ),
12145                ("run_timeout_seconds", self.options.run_timeout_seconds),
12146            ] {
12147                if value == Some(0) {
12148                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12149                        "{name} must be at least 1"
12150                    ))));
12151                }
12152            }
12153
12154            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12155                Ok(args) => args,
12156                Err(error) => return Poll::Ready(Err(error)),
12157            };
12158            let arguments = match encode_typed_envelope(
12159                &normalize_avro_arguments(args),
12160                &state.payload_codec,
12161            ) {
12162                Ok(arguments) => arguments,
12163                Err(error) => return Poll::Ready(Err(error)),
12164            };
12165            let mut command = json!({
12166                "type": "start_child_workflow",
12167                "workflow_type": self.workflow_type,
12168                "queue": self.options.task_queue,
12169                "parent_close_policy": self.options.parent_close_policy.as_str(),
12170                "arguments": arguments,
12171            });
12172            let object = command
12173                .as_object_mut()
12174                .expect("child workflow command is always an object");
12175            if let Some(policy) = &self.options.retry_policy {
12176                let mut retry_policy = serde_json::Map::new();
12177                if let Some(max_attempts) = policy.max_attempts {
12178                    if max_attempts == 0 {
12179                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12180                            "retry_policy.max_attempts must be at least 1".to_string(),
12181                        )));
12182                    }
12183                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12184                }
12185                if !policy.backoff_seconds.is_empty() {
12186                    retry_policy
12187                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12188                }
12189                if !policy.non_retryable_error_types.is_empty() {
12190                    retry_policy.insert(
12191                        "non_retryable_error_types".to_string(),
12192                        json!(policy.non_retryable_error_types),
12193                    );
12194                }
12195                if retry_policy.is_empty() {
12196                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12197                        "retry_policy must configure at least one field".to_string(),
12198                    )));
12199                }
12200                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12201            }
12202            if let Some(seconds) = self.options.execution_timeout_seconds {
12203                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12204            }
12205            if let Some(seconds) = self.options.run_timeout_seconds {
12206                object.insert("run_timeout_seconds".to_string(), json!(seconds));
12207            }
12208            apply_parallel_group_path(object, &self.parallel_group_path);
12209            state.commands.push(command);
12210            self.scheduled = true;
12211        }
12212
12213        Poll::Pending
12214    }
12215}
12216
12217impl Future for ChildWorkflowCall {
12218    type Output = Result<ChildWorkflowResult>;
12219
12220    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12221        match self.poll_avro_value(cx) {
12222            Poll::Ready(Ok(result)) => match result.result.into_json() {
12223                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12224                    parent: result.parent,
12225                    child: result.child,
12226                    child_workflow_type: result.child_workflow_type,
12227                    result: projected,
12228                })),
12229                Err(error) => Poll::Ready(Err(error)),
12230            },
12231            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12232            Poll::Pending => Poll::Pending,
12233        }
12234    }
12235}
12236
12237fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12238    Error::NonDeterministicReplay(ReplayFailure::new(
12239        "recorded_command_mismatch",
12240        Some(recorded.sequence()),
12241        Some(recorded.shape().to_string()),
12242        Some(actual.into()),
12243        "current workflow command does not match the recorded durable command sequence",
12244    ))
12245}
12246
12247pub struct SignalCall {
12248    ctx: WorkflowContext,
12249    signal_name: String,
12250    runtime_reserved_allowed: bool,
12251    opened_wait: bool,
12252    matched_pending: bool,
12253    parallel_group_path: Vec<ParallelGroupMetadata>,
12254}
12255
12256impl SignalCall {
12257    fn poll_avro_value(
12258        mut self: Pin<&mut Self>,
12259        _cx: &mut TaskContext<'_>,
12260    ) -> Poll<Result<Vec<AvroValue>>> {
12261        if self.matched_pending {
12262            return Poll::Pending;
12263        }
12264        if !self.runtime_reserved_allowed {
12265            if let Err(error) = validate_user_signal_name(&self.signal_name) {
12266                return Poll::Ready(Err(error));
12267            }
12268        }
12269
12270        let ctx = self.ctx.clone();
12271        let mut state = match ctx.state.lock() {
12272            Ok(state) => state,
12273            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12274        };
12275
12276        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12277            match recorded {
12278                RecordedCommand::SignalWait {
12279                    sequence,
12280                    signal_name,
12281                    value,
12282                    parallel_group_path,
12283                } => {
12284                    if let Err(error) = ensure_parallel_path_matches(
12285                        sequence,
12286                        parallel_group_path.as_deref(),
12287                        &self.parallel_group_path,
12288                    ) {
12289                        return Poll::Ready(Err(error));
12290                    }
12291                    if signal_name != self.signal_name {
12292                        return Poll::Ready(Err(Error::NonDeterministicReplay(
12293                            ReplayFailure::new(
12294                                "recorded_command_detail_mismatch",
12295                                Some(sequence),
12296                                Some(format!("signal wait:{signal_name}")),
12297                                Some(format!("signal wait:{}", self.signal_name)),
12298                                "recorded signal name differs from the current workflow command",
12299                            ),
12300                        )));
12301                    }
12302
12303                    state.command_cursor += 1;
12304                    if let Some(value) = value {
12305                        return Poll::Ready(Ok(value));
12306                    }
12307                    if state
12308                        .resume_signal
12309                        .as_ref()
12310                        .is_some_and(|signal| signal.signal_name == self.signal_name)
12311                    {
12312                        let signal = state
12313                            .resume_signal
12314                            .take()
12315                            .expect("matching resume signal is present");
12316                        return Poll::Ready(Ok(signal.arguments));
12317                    }
12318
12319                    state.matched_recorded_pending = true;
12320                    self.opened_wait = true;
12321                    self.matched_pending = true;
12322                    return Poll::Pending;
12323                }
12324                other => {
12325                    return Poll::Ready(Err(command_mismatch(
12326                        &other,
12327                        format!("signal wait:{}", self.signal_name),
12328                    )));
12329                }
12330            }
12331        }
12332
12333        if state
12334            .resume_signal
12335            .as_ref()
12336            .is_some_and(|signal| signal.signal_name == self.signal_name)
12337        {
12338            let signal = state
12339                .resume_signal
12340                .take()
12341                .expect("matching resume signal is present");
12342            return Poll::Ready(Ok(signal.arguments));
12343        }
12344
12345        if !self.opened_wait {
12346            let mut command = serde_json::Map::from_iter([
12347                ("type".to_string(), json!("open_signal_wait")),
12348                ("signal_name".to_string(), json!(self.signal_name)),
12349            ]);
12350            apply_parallel_group_path(&mut command, &self.parallel_group_path);
12351            state.commands.push(Value::Object(command));
12352            self.opened_wait = true;
12353        }
12354
12355        Poll::Pending
12356    }
12357}
12358
12359impl Future for SignalCall {
12360    type Output = Result<Vec<Value>>;
12361
12362    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12363        match self.poll_avro_value(cx) {
12364            Poll::Ready(Ok(values)) => Poll::Ready(
12365                values
12366                    .into_iter()
12367                    .map(AvroValue::into_json)
12368                    .collect::<Result<Vec<_>>>(),
12369            ),
12370            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12371            Poll::Pending => Poll::Pending,
12372        }
12373    }
12374}
12375
12376#[derive(Clone, Debug)]
12377pub struct ActivityContext {
12378    client: Client,
12379    pub task_id: String,
12380    pub activity_attempt_id: String,
12381    pub lease_owner: String,
12382    pub activity_type: String,
12383    pub attempt_number: u64,
12384    pub task_queue: String,
12385    pub worker_id: String,
12386}
12387
12388impl ActivityContext {
12389    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12390        self.client
12391            .heartbeat_activity_task(
12392                &self.task_id,
12393                &self.activity_attempt_id,
12394                &self.lease_owner,
12395                details,
12396            )
12397            .await
12398    }
12399}
12400
12401fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12402    validate_payload_codec(codec)?;
12403    match value {
12404        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12405            value, codec,
12406        )?)),
12407        None => Ok(AvroValue::Array(Vec::new())),
12408    }
12409}
12410
12411fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12412    let Some(signal_name) = task
12413        .signal_name
12414        .as_deref()
12415        .filter(|value| !value.is_empty())
12416    else {
12417        return Ok(None);
12418    };
12419    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12420    let AvroValue::Array(arguments) = decoded else {
12421        unreachable!("normalize_avro_arguments always returns an array");
12422    };
12423
12424    Ok(Some(ResumeSignal {
12425        signal_name: signal_name.to_string(),
12426        arguments,
12427    }))
12428}
12429
12430fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12431    validate_payload_codec(&task.payload_codec)?;
12432    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12433    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12434    for event in &task.history_events {
12435        validate_history_event_payloads(event, &task.payload_codec)?;
12436    }
12437    Ok(())
12438}
12439
12440fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12441    validate_payload_codec(&task.payload_codec)?;
12442    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12443}
12444
12445fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12446    validate_payload_codec(&task.payload_codec)?;
12447    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12448    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12449    for event in &task.history_events {
12450        validate_history_event_payloads(event, &task.payload_codec)?;
12451    }
12452
12453    let Some(export) = task.history_export.as_ref() else {
12454        return Ok(());
12455    };
12456    let export_codec = match export.get("payloads") {
12457        Some(payloads) => declared_payload_codec(payloads, "codec")?,
12458        None => None,
12459    }
12460    .unwrap_or(&task.payload_codec);
12461    validate_payload_codec(export_codec)?;
12462
12463    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12464        for event in events {
12465            let event_type = event
12466                .get("event_type")
12467                .or_else(|| event.get("type"))
12468                .and_then(Value::as_str)
12469                .unwrap_or_default();
12470            if let Some(payload) = event.get("payload") {
12471                validate_history_payloads(event_type, payload, export_codec)?;
12472            }
12473        }
12474    }
12475    for signal in export
12476        .get("signals")
12477        .and_then(Value::as_array)
12478        .into_iter()
12479        .flatten()
12480    {
12481        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12482        validate_payload_codec(codec)?;
12483        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12484    }
12485    for activity in export
12486        .get("activities")
12487        .and_then(Value::as_array)
12488        .into_iter()
12489        .flatten()
12490    {
12491        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12492        validate_payload_codec(codec)?;
12493        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12494        validate_optional_inbound_payload(activity.get("result"), codec)?;
12495    }
12496    Ok(())
12497}
12498
12499fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12500    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12501}
12502
12503fn validate_history_payloads(
12504    event_type: &str,
12505    payload: &Value,
12506    fallback_codec: &str,
12507) -> Result<()> {
12508    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12509    validate_payload_codec(codec)?;
12510    for field in history_payload_fields(event_type) {
12511        validate_optional_inbound_payload(payload.get(*field), codec)?;
12512    }
12513    Ok(())
12514}
12515
12516const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12517
12518fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12519    match event_type {
12520        "ActivityCompleted" => &["result"],
12521        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12522        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12523        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12524        "ChildRunCompleted" => &["result", "output"],
12525        "WorkflowCompleted" => &["output"],
12526        "ServiceCallStarted"
12527        | "ServiceCallCompleted"
12528        | "ServiceCallFailed"
12529        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12530        _ => &[],
12531    }
12532}
12533
12534fn signal_history_payload(payload: &Value) -> Option<&Value> {
12535    SIGNAL_HISTORY_PAYLOAD_FIELDS
12536        .iter()
12537        .find_map(|field| payload.get(*field))
12538}
12539
12540fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12541    match value.get(field) {
12542        None => Ok(None),
12543        Some(Value::String(codec)) => Ok(Some(codec)),
12544        Some(_) => Err(invalid_payload_envelope()),
12545    }
12546}
12547
12548fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12549    validate_payload_codec(codec)?;
12550    if let Some(value) = value.filter(|value| !value.is_null()) {
12551        decode_wire_avro_value(value, codec)?;
12552    }
12553    Ok(())
12554}
12555
12556fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12557    let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12558        invalid_recorded_history(
12559            "parallel_group_metadata_invalid",
12560            sequence,
12561            "non-empty parallel_group_id",
12562            &payload.to_string(),
12563            "parallel-group history is missing its stable identity",
12564        )
12565    })?;
12566    let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12567        invalid_recorded_history(
12568            "parallel_group_metadata_invalid",
12569            sequence,
12570            "activity, child, timer, signal, condition, or mixed group kind",
12571            &payload.to_string(),
12572            "parallel-group history is missing its group kind",
12573        )
12574    })?;
12575    if !matches!(
12576        kind.as_str(),
12577        "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12578    ) {
12579        return Err(invalid_recorded_history(
12580            "parallel_group_metadata_invalid",
12581            sequence,
12582            "activity, child, timer, signal, condition, or mixed group kind",
12583            &kind,
12584            "parallel-group history contains an unsupported group kind",
12585        ));
12586    }
12587    let base_sequence = payload
12588        .get("parallel_group_base_sequence")
12589        .and_then(value_as_u64)
12590        .filter(|value| *value > 0)
12591        .ok_or_else(|| {
12592            invalid_recorded_history(
12593                "parallel_group_metadata_invalid",
12594                sequence,
12595                "positive parallel_group_base_sequence",
12596                &payload.to_string(),
12597                "parallel-group history contains an invalid base sequence",
12598            )
12599        })?;
12600    let size = payload
12601        .get("parallel_group_size")
12602        .and_then(value_as_u64)
12603        .and_then(|value| usize::try_from(value).ok())
12604        .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12605        .ok_or_else(|| {
12606            invalid_recorded_history(
12607                "parallel_group_metadata_invalid",
12608                sequence,
12609                "bounded positive parallel_group_size",
12610                &payload.to_string(),
12611                "parallel-group history contains an invalid group size",
12612            )
12613        })?;
12614    let index = payload
12615        .get("parallel_group_index")
12616        .and_then(value_as_u64)
12617        .and_then(|value| usize::try_from(value).ok())
12618        .filter(|value| *value < size)
12619        .ok_or_else(|| {
12620            invalid_recorded_history(
12621                "parallel_group_metadata_invalid",
12622                sequence,
12623                "parallel_group_index within group bounds",
12624                &payload.to_string(),
12625                "parallel-group history contains an invalid member index",
12626            )
12627        })?;
12628    if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12629        return Err(invalid_recorded_history(
12630            "parallel_group_metadata_invalid",
12631            sequence,
12632            "base sequence plus member index equals workflow sequence",
12633            &payload.to_string(),
12634            "parallel-group path does not preserve durable workflow position",
12635        ));
12636    }
12637    let mode = payload
12638        .get("parallel_group_mode")
12639        .and_then(Value::as_str)
12640        .unwrap_or("all");
12641    if !matches!(mode, "all" | "select") {
12642        return Err(invalid_recorded_history(
12643            "parallel_group_metadata_invalid",
12644            sequence,
12645            "parallel group mode all or select",
12646            mode,
12647            "parallel-group history contains an unsupported group mode",
12648        ));
12649    }
12650    let expected_id = if mode == "select" {
12651        format!("select-calls:{base_sequence}:{size}")
12652    } else {
12653        format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12654    };
12655    if group_id != expected_id {
12656        return Err(invalid_recorded_history(
12657            "parallel_group_metadata_invalid",
12658            sequence,
12659            &expected_id,
12660            &group_id,
12661            "parallel-group history contains an incompatible stable group ID",
12662        ));
12663    }
12664    let selection_member_key = if mode == "select" {
12665        Some(selection_key_from_value(
12666            payload.get("selection_member_key"),
12667            sequence,
12668        )?)
12669    } else {
12670        None
12671    };
12672    let selection_member_index = if mode == "select" {
12673        Some(required_parallel_usize(
12674            payload,
12675            "selection_member_index",
12676            sequence,
12677        )?)
12678    } else {
12679        None
12680    };
12681    let selection_member_base_sequence = if mode == "select" {
12682        Some(
12683            payload
12684                .get("selection_member_base_sequence")
12685                .and_then(value_as_u64)
12686                .filter(|value| *value >= base_sequence)
12687                .ok_or_else(|| {
12688                    invalid_recorded_history(
12689                        "parallel_group_metadata_invalid",
12690                        sequence,
12691                        "selection member base within its group",
12692                        &payload.to_string(),
12693                        "selection history contains an invalid member base sequence",
12694                    )
12695                })?,
12696        )
12697    } else {
12698        None
12699    };
12700    let selection_member_size = if mode == "select" {
12701        let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12702        if member_size == 0 {
12703            return Err(invalid_recorded_history(
12704                "parallel_group_metadata_invalid",
12705                sequence,
12706                "positive selection member size",
12707                &payload.to_string(),
12708                "selection history contains an invalid member size",
12709            ));
12710        }
12711        Some(member_size)
12712    } else {
12713        None
12714    };
12715    let selection_member_kind = if mode == "select" {
12716        let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12717            invalid_recorded_history(
12718                "parallel_group_metadata_invalid",
12719                sequence,
12720                "selection member operation kind",
12721                &payload.to_string(),
12722                "selection history is missing its authored member kind",
12723            )
12724        })?;
12725        if !matches!(
12726            kind.as_str(),
12727            "activity" | "child" | "timer" | "signal" | "condition" | "group"
12728        ) {
12729            return Err(invalid_recorded_history(
12730                "parallel_group_metadata_invalid",
12731                sequence,
12732                "activity, child, timer, signal, condition, or group selection member kind",
12733                &kind,
12734                "selection history contains an unsupported member kind",
12735            ));
12736        }
12737        Some(kind)
12738    } else {
12739        None
12740    };
12741    if let (Some(member_base), Some(member_size)) =
12742        (selection_member_base_sequence, selection_member_size)
12743    {
12744        let member_end = member_base
12745            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12746            .ok_or_else(|| {
12747                invalid_recorded_history(
12748                    "parallel_group_metadata_invalid",
12749                    sequence,
12750                    "bounded selection member range",
12751                    &payload.to_string(),
12752                    "selection member range overflowed",
12753                )
12754            })?;
12755        let group_end = base_sequence
12756            .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12757            .unwrap_or(u64::MAX);
12758        if sequence < member_base || sequence >= member_end || member_end > group_end {
12759            return Err(invalid_recorded_history(
12760                "parallel_group_metadata_invalid",
12761                sequence,
12762                "workflow sequence within one bounded selection member",
12763                &payload.to_string(),
12764                "selection member range does not contain its durable leaf",
12765            ));
12766        }
12767    }
12768    Ok(ParallelGroupMetadata {
12769        parallel_group_id: group_id,
12770        parallel_group_kind: kind,
12771        parallel_group_base_sequence: base_sequence,
12772        parallel_group_size: size,
12773        parallel_group_index: index,
12774        parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12775        selection_member_key,
12776        selection_member_index,
12777        selection_member_base_sequence,
12778        selection_member_size,
12779        selection_member_kind,
12780    })
12781}
12782
12783fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12784    payload
12785        .get(field)
12786        .and_then(value_as_u64)
12787        .and_then(|value| usize::try_from(value).ok())
12788        .ok_or_else(|| {
12789            invalid_recorded_history(
12790                "parallel_group_metadata_invalid",
12791                sequence,
12792                &format!("non-negative integer {field}"),
12793                &payload.to_string(),
12794                "selection history contains invalid member metadata",
12795            )
12796        })
12797}
12798
12799fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12800    match value {
12801        Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12802        Some(value) => value_as_u64(value)
12803            .and_then(|value| usize::try_from(value).ok())
12804            .map(SelectionKey::Index)
12805            .ok_or_else(|| {
12806                invalid_recorded_history(
12807                    "selection_member_key_invalid",
12808                    sequence,
12809                    "non-empty string or non-negative integer member key",
12810                    &value.to_string(),
12811                    "selection history contains an invalid member key",
12812                )
12813            }),
12814        None => Err(invalid_recorded_history(
12815            "selection_member_key_missing",
12816            sequence,
12817            "selection_member_key",
12818            "<missing>",
12819            "selection history is missing its stable member key",
12820        )),
12821    }
12822}
12823
12824fn recorded_parallel_group_path(
12825    events: &[&HistoryEvent],
12826    sequence: u64,
12827) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12828    let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12829    for event in events {
12830        let payload = &event.payload;
12831        let has_metadata = payload.get("parallel_group_path").is_some()
12832            || payload.get("parallel_group_id").is_some()
12833            || payload.get("parallel_group_kind").is_some()
12834            || payload.get("parallel_group_base_sequence").is_some()
12835            || payload.get("parallel_group_size").is_some()
12836            || payload.get("parallel_group_index").is_some()
12837            || payload.get("parallel_group_mode").is_some()
12838            || payload.get("selection_member_key").is_some();
12839        if !has_metadata {
12840            continue;
12841        }
12842
12843        let top_level = recorded_parallel_group_entry(payload, sequence)?;
12844        let path = match payload.get("parallel_group_path") {
12845            None => vec![top_level.clone()],
12846            Some(Value::Array(entries)) if !entries.is_empty() => entries
12847                .iter()
12848                .map(|entry| recorded_parallel_group_entry(entry, sequence))
12849                .collect::<Result<Vec<_>>>()?,
12850            Some(value) => {
12851                return Err(invalid_recorded_history(
12852                    "parallel_group_metadata_invalid",
12853                    sequence,
12854                    "non-empty parallel_group_path list",
12855                    &value.to_string(),
12856                    "parallel-group history contains an invalid group path",
12857                ));
12858            }
12859        };
12860        if path.last() != Some(&top_level) {
12861            return Err(invalid_recorded_history(
12862                "parallel_group_metadata_invalid",
12863                sequence,
12864                &serde_json::to_string(&path.last()).unwrap_or_default(),
12865                &serde_json::to_string(&top_level).unwrap_or_default(),
12866                "parallel-group top-level fields do not match the innermost path entry",
12867            ));
12868        }
12869        if recorded.as_ref().is_some_and(|existing| existing != &path) {
12870            return Err(invalid_recorded_history(
12871                "parallel_group_history_conflict",
12872                sequence,
12873                &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12874                &serde_json::to_string(&path).unwrap_or_default(),
12875                "parallel-group metadata changed between scheduling and resolution history",
12876            ));
12877        }
12878        recorded = Some(path);
12879    }
12880    Ok(recorded)
12881}
12882
12883fn recorded_commands(
12884    events: &[HistoryEvent],
12885    fallback_codec: &str,
12886    parent: WorkflowIdentity,
12887) -> Result<Vec<RecordedCommand>> {
12888    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12889    let mut last_new_sequence = None;
12890
12891    for event in events {
12892        let is_activity = matches!(
12893            event.event_type.as_str(),
12894            "ActivityScheduled"
12895                | "ActivityStarted"
12896                | "ActivityHeartbeatRecorded"
12897                | "ActivityRetryScheduled"
12898                | "ActivityCompleted"
12899                | "ActivityFailed"
12900                | "ActivityCancelled"
12901                | "ActivityTimedOut"
12902        );
12903        let is_workflow_timer = matches!(
12904            event.event_type.as_str(),
12905            "TimerScheduled" | "TimerCancelled" | "TimerFired"
12906        ) && !is_internal_timer_event(event);
12907        let is_child_workflow = matches!(
12908            event.event_type.as_str(),
12909            "ChildWorkflowScheduled"
12910                | "ChildRunCompleted"
12911                | "ChildRunFailed"
12912                | "ChildRunCancelled"
12913                | "ChildRunTerminated"
12914        );
12915        let is_signal_wait = is_recorded_signal_wait_event(event);
12916        let is_condition_wait = is_recorded_condition_wait_event(event);
12917        let is_search_attributes = event.event_type == "SearchAttributesUpserted";
12918        let is_side_effect = event.event_type == "SideEffectRecorded";
12919        let is_version_marker = event.event_type == "VersionMarkerRecorded";
12920        let is_memo = event.event_type == "MemoUpserted";
12921        if !is_activity
12922            && !is_workflow_timer
12923            && !is_child_workflow
12924            && !is_signal_wait
12925            && !is_condition_wait
12926            && !is_search_attributes
12927            && !is_side_effect
12928            && !is_version_marker
12929            && !is_memo
12930        {
12931            continue;
12932        }
12933
12934        let sequence = durable_event_sequence(event).ok_or_else(|| {
12935            Error::NonDeterministicReplay(ReplayFailure::new(
12936                "durable_command_sequence_missing",
12937                None,
12938                Some("positive workflow sequence".to_string()),
12939                Some(event.event_type.clone()),
12940                "durable command history event has no workflow sequence",
12941            ))
12942        })?;
12943        if sequence == 0 {
12944            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12945                "durable_command_sequence_invalid",
12946                Some(sequence),
12947                Some("positive workflow sequence".to_string()),
12948                Some(sequence.to_string()),
12949                "durable command history uses an invalid workflow sequence",
12950            )));
12951        }
12952        if !events_by_sequence.contains_key(&sequence) {
12953            if let Some(previous) = last_new_sequence {
12954                if sequence < previous {
12955                    return Err(invalid_recorded_history(
12956                        "durable_command_sequence_mismatch",
12957                        sequence,
12958                        &format!("workflow sequence greater than {previous}"),
12959                        &sequence.to_string(),
12960                        "durable commands are not strictly ordered by their recorded workflow sequence",
12961                    ));
12962                }
12963            }
12964            last_new_sequence = Some(sequence);
12965        }
12966        events_by_sequence.entry(sequence).or_default().push(event);
12967    }
12968
12969    let commands: Vec<RecordedCommand> = events_by_sequence
12970        .into_iter()
12971        .map(|(sequence, sequence_events)| {
12972            let activity_events: Vec<_> = sequence_events
12973                .iter()
12974                .copied()
12975                .filter(|event| event.event_type.starts_with("Activity"))
12976                .collect();
12977            let timer_events: Vec<_> = sequence_events
12978                .iter()
12979                .copied()
12980                .filter(|event| event.event_type.starts_with("Timer"))
12981                .collect();
12982            let child_events: Vec<_> = sequence_events
12983                .iter()
12984                .copied()
12985                .filter(|event| {
12986                    event.event_type == "ChildWorkflowScheduled"
12987                        || event.event_type.starts_with("ChildRun")
12988                })
12989                .collect();
12990            let signal_wait_events: Vec<_> = sequence_events
12991                .iter()
12992                .copied()
12993                .filter(|event| is_recorded_signal_wait_event(event))
12994                .collect();
12995            let condition_wait_events: Vec<_> = sequence_events
12996                .iter()
12997                .copied()
12998                .filter(|event| is_recorded_condition_wait_event(event))
12999                .collect();
13000            let search_attribute_events: Vec<_> = sequence_events
13001                .iter()
13002                .copied()
13003                .filter(|event| event.event_type == "SearchAttributesUpserted")
13004                .collect();
13005            let side_effect_events: Vec<_> = sequence_events
13006                .iter()
13007                .copied()
13008                .filter(|event| event.event_type == "SideEffectRecorded")
13009                .collect();
13010            let version_marker_events: Vec<_> = sequence_events
13011                .iter()
13012                .copied()
13013                .filter(|event| event.event_type == "VersionMarkerRecorded")
13014                .collect();
13015            let memo_events: Vec<_> = sequence_events
13016                .iter()
13017                .copied()
13018                .filter(|event| event.event_type == "MemoUpserted")
13019                .collect();
13020
13021            let command_kind_count = usize::from(!activity_events.is_empty())
13022                + usize::from(!timer_events.is_empty())
13023                + usize::from(!child_events.is_empty())
13024                + usize::from(!signal_wait_events.is_empty())
13025                + usize::from(!condition_wait_events.is_empty())
13026                + usize::from(!search_attribute_events.is_empty())
13027                + usize::from(!side_effect_events.is_empty())
13028                + usize::from(!version_marker_events.is_empty())
13029                + usize::from(!memo_events.is_empty());
13030            if command_kind_count > 1 {
13031                let actual = [
13032                    (!activity_events.is_empty()).then_some("activity"),
13033                    (!timer_events.is_empty()).then_some("timer"),
13034                    (!child_events.is_empty()).then_some("child workflow"),
13035                    (!signal_wait_events.is_empty()).then_some("signal wait"),
13036                    (!condition_wait_events.is_empty()).then_some("condition wait"),
13037                    (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13038                    (!side_effect_events.is_empty()).then_some("side effect"),
13039                    (!version_marker_events.is_empty()).then_some("version marker"),
13040                    (!memo_events.is_empty()).then_some("memo upsert"),
13041                ]
13042                .into_iter()
13043                .flatten()
13044                .collect::<Vec<_>>()
13045                .join(" and ");
13046                return Err(invalid_recorded_history(
13047                    "durable_command_sequence_collision",
13048                    sequence,
13049                    "one durable command kind",
13050                    &actual,
13051                    "one workflow sequence records more than one durable command kind",
13052                ));
13053            }
13054
13055            if !activity_events.is_empty() {
13056                let parallel_group_path =
13057                    recorded_parallel_group_path(&activity_events, sequence)?;
13058                let scheduled_count = activity_events
13059                    .iter()
13060                    .filter(|event| event.event_type == "ActivityScheduled")
13061                    .count();
13062                if scheduled_count > 1 {
13063                    return Err(invalid_recorded_history(
13064                        "duplicate_activity_schedule",
13065                        sequence,
13066                        "at most one ActivityScheduled event",
13067                        "multiple ActivityScheduled events",
13068                        "activity history schedules more than one command at one workflow sequence",
13069                    ));
13070                }
13071                let activity_type = activity_events.iter().find_map(|event| {
13072                    event
13073                        .payload
13074                        .get("activity_type")
13075                        .or_else(|| event.payload.get("activity_name"))
13076                        .and_then(Value::as_str)
13077                        .map(str::to_string)
13078                });
13079                if activity_events.iter().filter_map(|event| {
13080                    event
13081                        .payload
13082                        .get("activity_type")
13083                        .or_else(|| event.payload.get("activity_name"))
13084                        .and_then(Value::as_str)
13085                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13086                    return Err(invalid_recorded_history(
13087                        "activity_identity_mismatch",
13088                        sequence,
13089                        activity_type.as_deref().unwrap_or("one activity identity"),
13090                        "conflicting activity identities",
13091                        "activity lifecycle events at one workflow sequence disagree on identity",
13092                    ));
13093                }
13094                let terminal: Vec<_> = activity_events
13095                    .iter()
13096                    .copied()
13097                    .filter(|event| {
13098                        matches!(
13099                            event.event_type.as_str(),
13100                            "ActivityCompleted"
13101                                | "ActivityFailed"
13102                                | "ActivityCancelled"
13103                                | "ActivityTimedOut"
13104                        )
13105                    })
13106                    .collect();
13107                let duplicate_delivery = terminal.first().is_some_and(|first| {
13108                    terminal.iter().all(|event| {
13109                        event.event_type == first.event_type && event.payload == first.payload
13110                    })
13111                });
13112                if terminal.len() > 1 && !duplicate_delivery {
13113                    return Err(invalid_recorded_history(
13114                        "duplicate_activity_terminal_event",
13115                        sequence,
13116                        "at most one terminal activity event",
13117                        "multiple terminal activity events",
13118                        "activity history settles one command more than once",
13119                    ));
13120                }
13121                let outcome = terminal
13122                    .first()
13123                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13124                    .transpose()?;
13125                let options = activity_events
13126                    .iter()
13127                    .find(|event| event.event_type == "ActivityScheduled")
13128                    .and_then(|event| event.payload.get("activity"))
13129                    .and_then(Value::as_object)
13130                    .map(|activity| RecordedActivityOptions {
13131                        task_queue: recorded_optional_string(activity, "queue"),
13132                        execution_mode: recorded_optional_string(activity, "execution_mode"),
13133                        retry_policy: recorded_activity_retry_snapshot(
13134                            activity.get("retry_policy"),
13135                        ),
13136                    });
13137                return Ok(RecordedCommand::Activity {
13138                    sequence,
13139                    activity_type,
13140                    options,
13141                    outcome,
13142                    parallel_group_path,
13143                });
13144            }
13145
13146            if !child_events.is_empty() {
13147                let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13148                let scheduled: Vec<_> = child_events
13149                    .iter()
13150                    .copied()
13151                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
13152                    .collect();
13153                if scheduled.len() != 1 {
13154                    return Err(invalid_recorded_history(
13155                        "child_workflow_schedule_missing_or_duplicate",
13156                        sequence,
13157                        "one ChildWorkflowScheduled event",
13158                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13159                        "child workflow replay requires exactly one recorded schedule event",
13160                    ));
13161                }
13162                let workflow_type = child_events.iter().find_map(|event| {
13163                    event
13164                        .payload
13165                        .get("child_workflow_type")
13166                        .or_else(|| event.payload.get("workflow_type"))
13167                        .and_then(Value::as_str)
13168                        .filter(|value| !value.is_empty())
13169                        .map(str::to_string)
13170                });
13171                if child_events
13172                    .iter()
13173                    .filter_map(|event| {
13174                        event
13175                            .payload
13176                            .get("child_workflow_type")
13177                            .or_else(|| event.payload.get("workflow_type"))
13178                            .and_then(Value::as_str)
13179                    })
13180                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
13181                {
13182                    return Err(invalid_recorded_history(
13183                        "child_workflow_identity_mismatch",
13184                        sequence,
13185                        workflow_type
13186                            .as_deref()
13187                            .unwrap_or("one child workflow type"),
13188                        "conflicting child workflow types",
13189                        "child workflow lifecycle events at one sequence disagree on type",
13190                    ));
13191                }
13192                let mut outcomes = child_workflow_outcomes(
13193                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13194                    fallback_codec,
13195                    parent.clone(),
13196                )?;
13197                let terminal_events = child_events
13198                    .iter()
13199                    .copied()
13200                    .filter(|event| event.event_type.starts_with("ChildRun"))
13201                    .collect::<Vec<_>>();
13202                let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13203                    terminal_events.iter().all(|event| {
13204                        event.event_type == first.event_type && event.payload == first.payload
13205                    })
13206                });
13207                if outcomes.len() > 1 && !duplicate_delivery {
13208                    return Err(invalid_recorded_history(
13209                        "duplicate_child_workflow_terminal_event",
13210                        sequence,
13211                        "at most one terminal child event",
13212                        "multiple terminal child events",
13213                        "child workflow history settles one command more than once",
13214                    ));
13215                }
13216                return Ok(RecordedCommand::ChildWorkflow {
13217                    sequence,
13218                    workflow_type,
13219                    outcome: outcomes.pop(),
13220                    parallel_group_path,
13221                });
13222            }
13223
13224            if !signal_wait_events.is_empty() {
13225                let opened: Vec<_> = signal_wait_events
13226                    .iter()
13227                    .copied()
13228                    .filter(|event| event.event_type == "SignalWaitOpened")
13229                    .collect();
13230                if opened.len() != 1 {
13231                    return Err(invalid_recorded_history(
13232                        "signal_wait_open_missing_or_duplicate",
13233                        sequence,
13234                        "one SignalWaitOpened event",
13235                        &format!("{} SignalWaitOpened events", opened.len()),
13236                        "signal replay requires exactly one canonical wait-open event",
13237                    ));
13238                }
13239
13240                let applied: Vec<_> = signal_wait_events
13241                    .iter()
13242                    .copied()
13243                    .filter(|event| event.event_type == "SignalApplied")
13244                    .collect();
13245                if applied.len() > 1 {
13246                    return Err(invalid_recorded_history(
13247                        "duplicate_signal_wait_apply",
13248                        sequence,
13249                        "at most one SignalApplied event",
13250                        "multiple SignalApplied events",
13251                        "signal history applies one durable wait more than once",
13252                    ));
13253                }
13254
13255                let signal_names = signal_wait_events
13256                    .iter()
13257                    .map(|event| required_signal_wait_name(event, sequence))
13258                    .collect::<Result<Vec<_>>>()?;
13259                let signal_name = signal_names
13260                    .first()
13261                    .expect("signal wait events are not empty")
13262                    .clone();
13263                if signal_names.iter().any(|candidate| candidate != &signal_name) {
13264                    return Err(invalid_recorded_history(
13265                        "signal_wait_identity_mismatch",
13266                        sequence,
13267                        &signal_name,
13268                        "conflicting signal names",
13269                        "signal wait lifecycle events at one workflow sequence disagree on identity",
13270                    ));
13271                }
13272                let value = applied
13273                    .first()
13274                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
13275                    .transpose()?;
13276                return Ok(RecordedCommand::SignalWait {
13277                    sequence,
13278                    signal_name,
13279                    value,
13280                    parallel_group_path: recorded_parallel_group_path(
13281                        &signal_wait_events,
13282                        sequence,
13283                    )?,
13284                });
13285            }
13286
13287            if !condition_wait_events.is_empty() {
13288                return recorded_condition_wait(
13289                    sequence,
13290                    &condition_wait_events,
13291                    events,
13292                );
13293            }
13294
13295            if !search_attribute_events.is_empty() {
13296                if search_attribute_events.len() != 1 {
13297                    return Err(invalid_recorded_history(
13298                        "duplicate_search_attribute_update",
13299                        sequence,
13300                        "one SearchAttributesUpserted event",
13301                        &format!(
13302                            "{} SearchAttributesUpserted events",
13303                            search_attribute_events.len()
13304                        ),
13305                        "search-attribute history records one workflow command more than once",
13306                    ));
13307                }
13308                let payload = &search_attribute_events[0].payload;
13309                let attributes = payload
13310                    .get("attributes")
13311                    .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13312                    .cloned()
13313                    .ok_or_else(|| {
13314                        invalid_recorded_history(
13315                            "search_attribute_update_missing",
13316                            sequence,
13317                            "non-empty attributes object",
13318                            "missing or invalid attributes",
13319                            "search-attribute history is missing its recorded mutation",
13320                        )
13321                    })?;
13322                let attribute_types =
13323                    recorded_search_attribute_types(payload, &attributes, sequence)?;
13324                return Ok(RecordedCommand::SearchAttributes {
13325                    sequence,
13326                    attributes,
13327                    attribute_types,
13328                });
13329            }
13330
13331            if !side_effect_events.is_empty() {
13332                if side_effect_events.len() != 1 {
13333                    return Err(invalid_recorded_history(
13334                        "duplicate_side_effect_record",
13335                        sequence,
13336                        "one SideEffectRecorded event",
13337                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
13338                        "side-effect history records one workflow command more than once",
13339                    ));
13340                }
13341                let event = side_effect_events[0];
13342                let result = event.payload.get("result").ok_or_else(|| {
13343                    invalid_recorded_history(
13344                        "side_effect_result_missing",
13345                        sequence,
13346                        "recorded result payload",
13347                        "missing result",
13348                        "side-effect history is missing its recorded value",
13349                    )
13350                })?;
13351                let has_published_envelope = result.as_str().is_some()
13352                    || result.as_object().is_some_and(|envelope| {
13353                        envelope.get("codec").and_then(Value::as_str).is_some()
13354                            && envelope.get("blob").and_then(Value::as_str).is_some()
13355                    });
13356                if !has_published_envelope {
13357                    return Err(invalid_recorded_history(
13358                        "side_effect_payload_malformed",
13359                        sequence,
13360                        "payload blob or {codec, blob} envelope",
13361                        &result.to_string(),
13362                        "side-effect history result does not use a published payload envelope",
13363                    ));
13364                }
13365                let codec = event
13366                    .payload
13367                    .get("payload_codec")
13368                    .and_then(Value::as_str)
13369                    .unwrap_or(fallback_codec);
13370                let value = decode_wire_avro_value(result, codec).map_err(|error| {
13371                    if error.to_string().contains("unsupported_payload_codec") {
13372                        return error;
13373                    }
13374
13375                    invalid_recorded_history(
13376                        "side_effect_payload_incompatible",
13377                        sequence,
13378                        &format!("valid {codec} payload envelope"),
13379                        &error.to_string(),
13380                        "side-effect history payload cannot be decoded with its recorded codec",
13381                    )
13382                })?;
13383                return Ok(RecordedCommand::SideEffect { sequence, value });
13384            }
13385
13386            if !version_marker_events.is_empty() {
13387                if version_marker_events.len() != 1 {
13388                    return Err(invalid_recorded_history(
13389                        "duplicate_version_marker_record",
13390                        sequence,
13391                        "one VersionMarkerRecorded event",
13392                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13393                        "version-marker history records one workflow command more than once",
13394                    ));
13395                }
13396                let payload = &version_marker_events[0].payload;
13397                let change_id = payload
13398                    .get("change_id")
13399                    .and_then(Value::as_str)
13400                    .filter(|value| !value.is_empty())
13401                    .map(str::to_string)
13402                    .ok_or_else(|| {
13403                        invalid_recorded_history(
13404                            "version_marker_field_missing",
13405                            sequence,
13406                            "non-empty change_id",
13407                            "missing or invalid change_id",
13408                            "version-marker history is missing its stable change ID",
13409                        )
13410                    })?;
13411                let version = required_version_i32(payload, "version", sequence)?;
13412                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13413                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13414                if min_supported > max_supported || version < min_supported || version > max_supported {
13415                    return Err(invalid_recorded_history(
13416                        "version_marker_history_range_invalid",
13417                        sequence,
13418                        "min_supported <= version <= max_supported",
13419                        &format!("{min_supported} <= {version} <= {max_supported}"),
13420                        "recorded version marker contains an internally incompatible range",
13421                    ));
13422                }
13423                return Ok(RecordedCommand::VersionMarker {
13424                    sequence,
13425                    change_id,
13426                    version,
13427                });
13428            }
13429
13430            if !memo_events.is_empty() {
13431                if memo_events.len() != 1 {
13432                    return Err(invalid_recorded_history(
13433                        "duplicate_memo_upsert_record",
13434                        sequence,
13435                        "one MemoUpserted event",
13436                        &format!("{} MemoUpserted events", memo_events.len()),
13437                        "memo history records one workflow update more than once",
13438                    ));
13439                }
13440                let payload = &memo_events[0].payload;
13441                let entries = payload.get("entries").cloned().ok_or_else(|| {
13442                    invalid_recorded_history(
13443                        "memo_entries_missing",
13444                        sequence,
13445                        "memo entries object",
13446                        "missing entries",
13447                        "MemoUpserted history is missing replay identity entries",
13448                    )
13449                })?;
13450                let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13451                    invalid_recorded_history(
13452                        "memo_entries_invalid",
13453                        sequence,
13454                        "valid canonical memo entries",
13455                        &error.to_string(),
13456                        "MemoUpserted history contains invalid replay identity entries",
13457                    )
13458                })?;
13459                let merged = payload.get("merged").cloned().ok_or_else(|| {
13460                    invalid_recorded_history(
13461                        "memo_merged_projection_missing",
13462                        sequence,
13463                        "merged memo projection",
13464                        "missing merged",
13465                        "MemoUpserted history is missing its merged projection",
13466                    )
13467                })?;
13468                decode_memo_history_map(&merged, false).map_err(|error| {
13469                    invalid_recorded_history(
13470                        "memo_merged_projection_invalid",
13471                        sequence,
13472                        "valid merged memo projection",
13473                        &error.to_string(),
13474                        "MemoUpserted history contains an invalid merged projection",
13475                    )
13476                })?;
13477
13478                return Ok(RecordedCommand::Memo { sequence, entries });
13479            }
13480            let scheduled: Vec<_> = timer_events
13481                .iter()
13482                .copied()
13483                .filter(|event| event.event_type == "TimerScheduled")
13484                .collect();
13485            let fired: Vec<_> = timer_events
13486                .iter()
13487                .copied()
13488                .filter(|event| event.event_type == "TimerFired")
13489                .collect();
13490            if scheduled.len() != 1 {
13491                return Err(invalid_recorded_history(
13492                    "timer_schedule_missing_or_duplicate",
13493                    sequence,
13494                    "one TimerScheduled event",
13495                    &format!("{} TimerScheduled events", scheduled.len()),
13496                    "timer replay requires exactly one recorded schedule event",
13497                ));
13498            }
13499            if fired.len() > 1 {
13500                return Err(invalid_recorded_history(
13501                    "duplicate_timer_fire",
13502                    sequence,
13503                    "at most one TimerFired event",
13504                    "multiple TimerFired events",
13505                    "timer history contains more than one fire event for a workflow sequence",
13506                ));
13507            }
13508
13509            let scheduled = scheduled[0];
13510            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13511            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13512            if let Some(fired) = fired.first() {
13513                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13514                if fired_timer_id != timer_id {
13515                    return Err(invalid_recorded_history(
13516                        "timer_identity_mismatch",
13517                        sequence,
13518                        &timer_id,
13519                        &fired_timer_id,
13520                        "TimerFired does not correspond to the recorded TimerScheduled event",
13521                    ));
13522                }
13523                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13524                if fired_delay != delay_seconds {
13525                    return Err(invalid_recorded_history(
13526                        "timer_history_delay_mismatch",
13527                        sequence,
13528                        &delay_seconds.to_string(),
13529                        &fired_delay.to_string(),
13530                        "TimerScheduled and TimerFired record different delays",
13531                    ));
13532                }
13533            }
13534
13535            Ok(RecordedCommand::Timer {
13536                sequence,
13537                delay_seconds,
13538                fired: !fired.is_empty(),
13539                parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13540            })
13541        })
13542        .collect::<Result<_>>()?;
13543
13544    let mut marker_sequences = HashMap::new();
13545    for command in &commands {
13546        if let RecordedCommand::VersionMarker {
13547            sequence,
13548            change_id,
13549            ..
13550        } = command
13551        {
13552            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13553                return Err(invalid_recorded_history(
13554                    "duplicate_version_marker",
13555                    *sequence,
13556                    &format!("one marker for change ID {change_id:?}"),
13557                    &format!("markers at sequences {first_sequence} and {sequence}"),
13558                    "workflow history contains duplicate markers for one stable change ID",
13559                ));
13560            }
13561        }
13562    }
13563
13564    Ok(commands)
13565}
13566
13567fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13568    payload
13569        .get(field)
13570        .and_then(Value::as_i64)
13571        .and_then(|value| i32::try_from(value).ok())
13572        .ok_or_else(|| {
13573            invalid_recorded_history(
13574                "version_marker_field_missing",
13575                sequence,
13576                &format!("integer {field}"),
13577                "missing or out-of-range integer",
13578                "version-marker history is missing a required integer field",
13579            )
13580        })
13581}
13582
13583fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13584    event
13585        .payload
13586        .get("sequence")
13587        .or_else(|| event.payload.get("workflow_sequence"))
13588        .or_else(|| event.raw.get("sequence"))
13589        .or_else(|| event.raw.get("workflow_sequence"))
13590        .and_then(value_as_u64)
13591}
13592
13593fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13594    matches!(
13595        event
13596            .payload
13597            .get("timer_kind")
13598            .or_else(|| event.raw.get("timer_kind"))
13599            .and_then(Value::as_str),
13600        Some("condition_timeout" | "signal_timeout")
13601    )
13602}
13603
13604fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13605    matches!(
13606        event.event_type.as_str(),
13607        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13608    )
13609}
13610
13611fn recorded_condition_wait(
13612    sequence: u64,
13613    condition_events: &[&HistoryEvent],
13614    all_events: &[HistoryEvent],
13615) -> Result<RecordedCommand> {
13616    let opened = condition_events
13617        .iter()
13618        .copied()
13619        .filter(|event| event.event_type == "ConditionWaitOpened")
13620        .collect::<Vec<_>>();
13621    if opened.len() != 1 {
13622        return Err(invalid_recorded_history(
13623            "condition_wait_open_missing_or_duplicate",
13624            sequence,
13625            "one ConditionWaitOpened event",
13626            &format!("{} ConditionWaitOpened events", opened.len()),
13627            "condition replay requires exactly one canonical wait-open event",
13628        ));
13629    }
13630    let terminal = condition_events
13631        .iter()
13632        .copied()
13633        .filter(|event| {
13634            matches!(
13635                event.event_type.as_str(),
13636                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13637            )
13638        })
13639        .collect::<Vec<_>>();
13640    if terminal.len() > 1 {
13641        return Err(invalid_recorded_history(
13642            "duplicate_condition_wait_terminal_event",
13643            sequence,
13644            "at most one condition terminal event",
13645            "multiple condition terminal events",
13646            "condition history settles one durable wait more than once",
13647        ));
13648    }
13649
13650    let opened = opened[0];
13651    let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13652    let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13653    for event in condition_events
13654        .iter()
13655        .copied()
13656        .filter(|event| !std::ptr::eq(*event, opened))
13657    {
13658        let event_wait_id = required_condition_wait_id(event, sequence)?;
13659        if event_wait_id != condition_wait_id {
13660            return Err(invalid_recorded_history(
13661                "condition_wait_id_mismatch",
13662                sequence,
13663                &condition_wait_id,
13664                &event_wait_id,
13665                "condition lifecycle events at one sequence disagree on wait identity",
13666            ));
13667        }
13668        let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13669        if event_occurrence_id != occurrence_id {
13670            return Err(invalid_recorded_history(
13671                "condition_wait_occurrence_history_mismatch",
13672                sequence,
13673                &occurrence_id,
13674                &event_occurrence_id,
13675                "condition lifecycle events at one sequence disagree on authored occurrence identity",
13676            ));
13677        }
13678    }
13679
13680    let condition_key = optional_non_empty_history_string(opened, "condition_key");
13681    let predicate_identity = opened
13682        .payload
13683        .get("condition_definition_fingerprint")
13684        .and_then(Value::as_str)
13685        .filter(|value| !value.is_empty())
13686        .map(str::to_string)
13687        .ok_or_else(|| {
13688            invalid_recorded_history(
13689                "condition_wait_predicate_fingerprint_missing",
13690                sequence,
13691                "non-empty condition_definition_fingerprint",
13692                &opened.event_type,
13693                "canonical condition history is missing its predicate identity",
13694            )
13695        })?;
13696    let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13697    for event in condition_events
13698        .iter()
13699        .copied()
13700        .filter(|event| !std::ptr::eq(*event, opened))
13701    {
13702        for (field, opened_value) in [
13703            ("condition_key", condition_key.as_deref()),
13704            (
13705                "condition_definition_fingerprint",
13706                Some(predicate_identity.as_str()),
13707            ),
13708        ] {
13709            if let Some(value) = optional_non_empty_history_string(event, field) {
13710                if opened_value.is_some_and(|opened_value| opened_value != value) {
13711                    return Err(invalid_recorded_history(
13712                        "condition_wait_definition_history_mismatch",
13713                        sequence,
13714                        opened_value.unwrap_or_default(),
13715                        &value,
13716                        "condition lifecycle events disagree on the recorded definition",
13717                    ));
13718                }
13719            }
13720        }
13721        if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13722            if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13723                return Err(invalid_recorded_history(
13724                    "condition_wait_definition_history_mismatch",
13725                    sequence,
13726                    &format!("{}s", timeout_seconds.unwrap_or_default()),
13727                    &format!("{event_timeout}s"),
13728                    "condition lifecycle events disagree on the recorded timeout",
13729                ));
13730            }
13731        }
13732    }
13733
13734    let timeout_timer_events = all_events
13735        .iter()
13736        .filter(|event| {
13737            matches!(
13738                event.event_type.as_str(),
13739                "TimerScheduled" | "TimerCancelled" | "TimerFired"
13740            ) && event.payload.get("timer_kind").and_then(Value::as_str)
13741                == Some("condition_timeout")
13742                && event
13743                    .payload
13744                    .get("condition_wait_id")
13745                    .and_then(Value::as_str)
13746                    == Some(condition_wait_id.as_str())
13747        })
13748        .collect::<Vec<_>>();
13749    let scheduled = timeout_timer_events
13750        .iter()
13751        .copied()
13752        .filter(|event| event.event_type == "TimerScheduled")
13753        .collect::<Vec<_>>();
13754    let fired = timeout_timer_events
13755        .iter()
13756        .copied()
13757        .filter(|event| event.event_type == "TimerFired")
13758        .collect::<Vec<_>>();
13759    if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13760        return Err(invalid_recorded_history(
13761            "condition_wait_timeout_history_invalid",
13762            sequence,
13763            "one timeout schedule and at most one fire",
13764            &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13765            "condition timeout history has a missing or duplicate lifecycle event",
13766        ));
13767    }
13768    if let Some(scheduled) = scheduled.first() {
13769        let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13770        let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13771        if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13772            return Err(invalid_recorded_history(
13773                "condition_wait_timeout_delay_mismatch",
13774                sequence,
13775                &format!("{}s", timeout_seconds.unwrap_or_default()),
13776                &format!("{delay_seconds}s"),
13777                "condition timeout timer differs from the wait definition",
13778            ));
13779        }
13780        if let Some(fired) = fired.first() {
13781            let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13782            let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13783            if fired_timer_id != timer_id || fired_delay != delay_seconds {
13784                return Err(invalid_recorded_history(
13785                    "condition_wait_timeout_identity_mismatch",
13786                    sequence,
13787                    &format!("{timer_id}:{delay_seconds}s"),
13788                    &format!("{fired_timer_id}:{fired_delay}s"),
13789                    "condition timeout fire does not match its durable schedule",
13790                ));
13791            }
13792        }
13793    }
13794
13795    let result = terminal.first().map(|event| {
13796        if event.event_type == "ConditionWaitTimedOut" {
13797            ConditionWaitResult::TimedOut
13798        } else {
13799            ConditionWaitResult::Satisfied
13800        }
13801    });
13802    let result = if !fired.is_empty() {
13803        if result == Some(ConditionWaitResult::Satisfied) {
13804            return Err(invalid_recorded_history(
13805                "condition_wait_terminal_conflict",
13806                sequence,
13807                "one satisfied or timed-out outcome",
13808                "satisfied event and fired timeout",
13809                "condition history records conflicting terminal outcomes",
13810            ));
13811        }
13812        Some(ConditionWaitResult::TimedOut)
13813    } else {
13814        result
13815    };
13816
13817    Ok(RecordedCommand::ConditionWait {
13818        sequence,
13819        occurrence_id,
13820        condition_key,
13821        predicate_identity,
13822        timeout_seconds,
13823        result,
13824        parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13825    })
13826}
13827
13828fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13829    event
13830        .payload
13831        .get("condition_wait_occurrence_id")
13832        .and_then(Value::as_str)
13833        .filter(|value| !value.is_empty())
13834        .map(str::to_string)
13835        .ok_or_else(|| {
13836            invalid_recorded_history(
13837                "condition_wait_occurrence_id_missing",
13838                sequence,
13839                "non-empty condition_wait_occurrence_id",
13840                &event.event_type,
13841                "condition history is missing authored occurrence identity",
13842            )
13843        })
13844}
13845
13846fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13847    event
13848        .payload
13849        .get("condition_wait_id")
13850        .and_then(Value::as_str)
13851        .filter(|value| !value.is_empty())
13852        .map(str::to_string)
13853        .ok_or_else(|| {
13854            invalid_recorded_history(
13855                "condition_wait_id_missing",
13856                sequence,
13857                "non-empty condition_wait_id",
13858                &event.event_type,
13859                "canonical condition history is missing its durable wait identity",
13860            )
13861        })
13862}
13863
13864fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13865    event
13866        .payload
13867        .get(field)
13868        .and_then(Value::as_str)
13869        .filter(|value| !value.is_empty())
13870        .map(str::to_string)
13871}
13872
13873fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13874    match event.payload.get(field) {
13875        None | Some(Value::Null) => Ok(None),
13876        Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13877            invalid_recorded_history(
13878                "condition_wait_definition_invalid",
13879                sequence,
13880                &format!("non-negative integer {field}"),
13881                &value.to_string(),
13882                "condition history contains an invalid numeric definition field",
13883            )
13884        }),
13885    }
13886}
13887
13888fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13889    event
13890        .payload
13891        .get("signal_name")
13892        .or_else(|| event.raw.get("signal_name"))
13893        .and_then(Value::as_str)
13894        .filter(|value| !value.is_empty())
13895        .map(str::to_string)
13896        .ok_or_else(|| {
13897            invalid_recorded_history(
13898                "signal_wait_name_missing",
13899                sequence,
13900                "non-empty signal_name",
13901                &event.event_type,
13902                "canonical signal-wait history is missing its signal identity",
13903            )
13904        })
13905}
13906
13907fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13908    matches!(
13909        event.event_type.as_str(),
13910        "SignalWaitOpened" | "SignalApplied"
13911    )
13912}
13913
13914fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
13915    event
13916        .payload
13917        .get(field)
13918        .and_then(Value::as_str)
13919        .filter(|value| !value.is_empty())
13920        .map(str::to_string)
13921        .ok_or_else(|| {
13922            invalid_recorded_history(
13923                "timer_history_field_missing",
13924                sequence,
13925                field,
13926                &event.event_type,
13927                "timer history is missing a required identity field",
13928            )
13929        })
13930}
13931
13932fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
13933    event
13934        .payload
13935        .get(field)
13936        .and_then(value_as_u64)
13937        .ok_or_else(|| {
13938            invalid_recorded_history(
13939                "timer_history_field_missing",
13940                sequence,
13941                field,
13942                &event.event_type,
13943                "timer history is missing a required numeric field",
13944            )
13945        })
13946}
13947
13948fn recorded_search_attribute_types(
13949    payload: &Value,
13950    attributes: &Value,
13951    sequence: u64,
13952) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
13953    let Some(raw_types) = payload.get("attribute_types") else {
13954        // This is the explicit compatibility rule for histories recorded
13955        // before typed identity was persisted. Values still constrain replay;
13956        // the unknown type snapshot does not assert a typed match.
13957        return Ok(RecordedSnapshotValue::Unknown);
13958    };
13959    let Some(raw_types) = raw_types.as_object() else {
13960        return Err(invalid_recorded_history(
13961            "search_attribute_types_malformed",
13962            sequence,
13963            "canonical attribute type map",
13964            &raw_types.to_string(),
13965            "search-attribute history contains malformed type identity",
13966        ));
13967    };
13968    let attribute_keys = attributes
13969        .as_object()
13970        .expect("recorded search attributes were validated as an object");
13971    let mut types = BTreeMap::new();
13972    for (key, value) in raw_types {
13973        let Some(attribute_type) = value.as_str() else {
13974            return Err(invalid_recorded_history(
13975                "search_attribute_types_malformed",
13976                sequence,
13977                "canonical string type name",
13978                &value.to_string(),
13979                "search-attribute history contains a non-string type identity",
13980            ));
13981        };
13982        if !attribute_keys.contains_key(key)
13983            || !matches!(
13984                attribute_type,
13985                "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
13986            )
13987        {
13988            return Err(invalid_recorded_history(
13989                "search_attribute_types_malformed",
13990                sequence,
13991                "canonical types for keys present in attributes",
13992                &format!("{key}:{attribute_type}"),
13993                "search-attribute history contains unsupported or orphaned type identity",
13994            ));
13995        }
13996        types.insert(key.clone(), attribute_type.to_string());
13997    }
13998    Ok(RecordedSnapshotValue::Known(types))
13999}
14000
14001fn invalid_recorded_history(
14002    reason: &str,
14003    sequence: u64,
14004    expected: &str,
14005    actual: &str,
14006    message: &str,
14007) -> Error {
14008    Error::NonDeterministicReplay(ReplayFailure::new(
14009        reason,
14010        Some(sequence),
14011        Some(expected.to_string()),
14012        Some(actual.to_string()),
14013        message,
14014    ))
14015}
14016
14017type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
14018
14019fn activity_outcome(
14020    event: &HistoryEvent,
14021    fallback_codec: &str,
14022    recorded_activity_type: Option<String>,
14023) -> Result<ActivityOutcome> {
14024    if event.event_type == "ActivityCompleted" {
14025        let codec = event
14026            .payload
14027            .get("payload_codec")
14028            .and_then(Value::as_str)
14029            .unwrap_or(fallback_codec);
14030        return Ok(Ok(decode_wire_avro_value(
14031            event.payload.get("result").unwrap_or(&Value::Null),
14032            codec,
14033        )?));
14034    }
14035
14036    let payload = &event.payload;
14037    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14038        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14039        "ActivityCancelled" => (
14040            ActivityFailureKind::Cancelled,
14041            "cancelled",
14042            "activity was cancelled",
14043        ),
14044        "ActivityTimedOut" => (
14045            ActivityFailureKind::TimedOut,
14046            "timeout",
14047            "activity timed out",
14048        ),
14049        _ => unreachable!("activity_outcome is called only for terminal activity events"),
14050    };
14051    let exception = payload
14052        .get("exception")
14053        .filter(|value| !value.is_null())
14054        .cloned();
14055    let failure_category = payload_string(payload, "failure_category");
14056    let timeout_kind = payload_string(payload, "timeout_kind");
14057    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14058        ActivityFailureKind::Failed => failure_category
14059            .clone()
14060            .unwrap_or_else(|| fallback_reason.to_string()),
14061        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14062        ActivityFailureKind::TimedOut => timeout_kind
14063            .clone()
14064            .unwrap_or_else(|| fallback_reason.to_string()),
14065    });
14066    let message = payload_string(payload, "message")
14067        .or_else(|| {
14068            exception
14069                .as_ref()
14070                .and_then(|value| payload_string(value, "message"))
14071        })
14072        .unwrap_or_else(|| fallback_message.to_string());
14073
14074    Ok(Err(ActivityFailure {
14075        kind,
14076        reason,
14077        message,
14078        activity_execution_id: payload_string(payload, "activity_execution_id"),
14079        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14080        activity_type: payload_string(payload, "activity_type")
14081            .or_else(|| payload_string(payload, "activity_name"))
14082            .or(recorded_activity_type),
14083        activity_class: payload_string(payload, "activity_class"),
14084        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14085        failure_id: payload_string(payload, "failure_id"),
14086        failure_category,
14087        timeout_kind,
14088        non_retryable: payload
14089            .get("non_retryable")
14090            .and_then(Value::as_bool)
14091            .unwrap_or(false),
14092        exception_type: payload_string(payload, "exception_type").or_else(|| {
14093            exception
14094                .as_ref()
14095                .and_then(|value| payload_string(value, "type"))
14096        }),
14097        exception_class: payload_string(payload, "exception_class").or_else(|| {
14098            exception
14099                .as_ref()
14100                .and_then(|value| payload_string(value, "class"))
14101        }),
14102        code: payload
14103            .get("code")
14104            .filter(|value| !value.is_null())
14105            .cloned(),
14106        exception,
14107    }))
14108}
14109
14110type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14111
14112fn child_workflow_outcomes(
14113    events: &[HistoryEvent],
14114    fallback_codec: &str,
14115    parent: WorkflowIdentity,
14116) -> Result<Vec<ChildWorkflowOutcome>> {
14117    let mut outcomes = Vec::new();
14118
14119    for event in events {
14120        let kind = match event.event_type.as_str() {
14121            "ChildRunCompleted" => None,
14122            "ChildRunFailed" => Some((
14123                ChildWorkflowFailureKind::Failed,
14124                "child_workflow",
14125                "child workflow failed",
14126            )),
14127            "ChildRunCancelled" => Some((
14128                ChildWorkflowFailureKind::Cancelled,
14129                "cancelled",
14130                "child workflow was cancelled",
14131            )),
14132            "ChildRunTerminated" => Some((
14133                ChildWorkflowFailureKind::Terminated,
14134                "terminated",
14135                "child workflow was terminated",
14136            )),
14137            _ => continue,
14138        };
14139        let payload = &event.payload;
14140        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14141        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14142        let child_workflow_type = payload_string(payload, "child_workflow_type");
14143
14144        if let Some((kind, reason, fallback_message)) = kind {
14145            let exception = payload
14146                .get("exception")
14147                .filter(|value| !value.is_null())
14148                .cloned();
14149            let message = payload_string(payload, "message")
14150                .or_else(|| {
14151                    exception
14152                        .as_ref()
14153                        .and_then(|value| payload_string(value, "message"))
14154                })
14155                .unwrap_or_else(|| fallback_message.to_string());
14156            let exception_type = payload_string(payload, "exception_type").or_else(|| {
14157                exception
14158                    .as_ref()
14159                    .and_then(|value| payload_string(value, "type"))
14160            });
14161            let exception_class = payload_string(payload, "exception_class").or_else(|| {
14162                exception
14163                    .as_ref()
14164                    .and_then(|value| payload_string(value, "class"))
14165            });
14166            outcomes.push(Err(ChildWorkflowFailure {
14167                kind,
14168                reason: reason.to_string(),
14169                message,
14170                parent_workflow_id: parent.workflow_id.clone(),
14171                parent_workflow_run_id: parent.run_id.clone(),
14172                child_workflow_id,
14173                child_workflow_run_id,
14174                child_workflow_type,
14175                failure_id: payload_string(payload, "failure_id"),
14176                failure_category: payload_string(payload, "failure_category"),
14177                exception_type,
14178                exception_class,
14179                non_retryable: payload
14180                    .get("non_retryable")
14181                    .and_then(Value::as_bool)
14182                    .unwrap_or(false),
14183                code: payload
14184                    .get("code")
14185                    .filter(|value| !value.is_null())
14186                    .cloned(),
14187                exception,
14188            }));
14189            continue;
14190        }
14191
14192        let codec = payload
14193            .get("payload_codec")
14194            .and_then(Value::as_str)
14195            .unwrap_or(fallback_codec);
14196        let result = payload
14197            .get("result")
14198            .or_else(|| payload.get("output"))
14199            .unwrap_or(&Value::Null);
14200        outcomes.push(Ok(ChildWorkflowAvroResult {
14201            parent: parent.clone(),
14202            child: WorkflowIdentity {
14203                workflow_id: child_workflow_id,
14204                run_id: child_workflow_run_id,
14205            },
14206            child_workflow_type,
14207            result: decode_wire_avro_value(result, codec)?,
14208        }));
14209    }
14210
14211    Ok(outcomes)
14212}
14213
14214fn payload_string(payload: &Value, key: &str) -> Option<String> {
14215    payload
14216        .get(key)
14217        .and_then(Value::as_str)
14218        .filter(|value| !value.is_empty())
14219        .map(str::to_string)
14220}
14221
14222fn workflow_failure_command(error: &Error) -> Value {
14223    let (exception_type, exception_class, properties) = match error {
14224        Error::ActivityFailed(failure) => (
14225            match failure.kind {
14226                ActivityFailureKind::Failed => "ActivityFailed",
14227                ActivityFailureKind::Cancelled => "ActivityCancelled",
14228                ActivityFailureKind::TimedOut => "ActivityTimedOut",
14229            },
14230            "durable_workflow::ActivityFailure",
14231            json!({
14232                "reason": failure.reason,
14233                "activity_execution_id": failure.activity_execution_id,
14234                "activity_attempt_id": failure.activity_attempt_id,
14235                "activity_type": failure.activity_type,
14236                "activity_class": failure.activity_class,
14237                "attempt_number": failure.attempt_number,
14238                "failure_id": failure.failure_id,
14239                "failure_category": failure.failure_category,
14240                "timeout_kind": failure.timeout_kind,
14241                "activity_non_retryable": failure.non_retryable,
14242                "activity_exception_type": failure.exception_type,
14243                "activity_exception_class": failure.exception_class,
14244                "activity_code": failure.code,
14245                "activity_exception": failure.exception,
14246            }),
14247        ),
14248        Error::ChildWorkflowFailed(failure) => (
14249            match failure.kind {
14250                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14251                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14252                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14253            },
14254            "durable_workflow::ChildWorkflowFailure",
14255            json!({
14256                "reason": failure.reason,
14257                "parent_workflow_id": failure.parent_workflow_id,
14258                "parent_workflow_run_id": failure.parent_workflow_run_id,
14259                "child_workflow_id": failure.child_workflow_id,
14260                "child_workflow_run_id": failure.child_workflow_run_id,
14261                "child_workflow_type": failure.child_workflow_type,
14262                "failure_id": failure.failure_id,
14263                "failure_category": failure.failure_category,
14264                "child_exception_type": failure.exception_type,
14265                "child_exception_class": failure.exception_class,
14266                "child_non_retryable": failure.non_retryable,
14267                "child_code": failure.code,
14268                "child_exception": failure.exception,
14269            }),
14270        ),
14271        Error::ParallelFailed(failure) => (
14272            "ParallelFailed",
14273            "durable_workflow::ParallelFailure",
14274            json!({
14275                "parallel_group_id": failure.group_id,
14276                "parallel_member_path": failure.member_path,
14277                "parallel_group_path": failure.group_path,
14278                "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14279                "cause_type": workflow_error_type(&failure.cause),
14280                "cause_message": failure.cause.to_string(),
14281            }),
14282        ),
14283        Error::SagaCompensationFailed(failure) => (
14284            "SagaCompensationFailed",
14285            "durable_workflow::SagaCompensationFailure",
14286            json!({
14287                "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14288                "initiating_failure_message": failure.initiating_failure.to_string(),
14289                "compensation_activity_type": failure.compensation_activity_type,
14290                "compensation_registration_order": failure.compensation_registration_order,
14291                "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14292                "compensation_failure_message": failure.compensation_failure.to_string(),
14293            }),
14294        ),
14295        Error::WorkflowCancellationRequested(_) => (
14296            "WorkflowCancellationRequested",
14297            "durable_workflow::WorkflowCancellationRequested",
14298            json!({"reason": "cancelled"}),
14299        ),
14300        Error::NonDeterministicReplay(_) => (
14301            "NonDeterministicReplay",
14302            "durable_workflow::Error",
14303            Value::Null,
14304        ),
14305        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14306    };
14307    let non_retryable = match error {
14308        Error::ActivityFailed(failure) => failure.non_retryable,
14309        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14310        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14311        Error::SagaCompensationFailed(failure) => {
14312            workflow_error_non_retryable(&failure.compensation_failure)
14313        }
14314        Error::WorkflowCancellationRequested(_) => true,
14315        Error::NonDeterministicReplay(_) => true,
14316        _ => false,
14317    };
14318
14319    json!({
14320        "type": "fail_workflow",
14321        "message": error.to_string(),
14322        "exception_type": exception_type,
14323        "exception_class": exception_class,
14324        "non_retryable": non_retryable,
14325        "exception": {
14326            "type": exception_type,
14327            "class": exception_class,
14328            "message": error.to_string(),
14329            "properties": properties,
14330        }
14331    })
14332}
14333
14334fn workflow_error_type(error: &Error) -> &'static str {
14335    match error {
14336        Error::ActivityFailed(failure) => match failure.kind {
14337            ActivityFailureKind::Failed => "ActivityFailed",
14338            ActivityFailureKind::Cancelled => "ActivityCancelled",
14339            ActivityFailureKind::TimedOut => "ActivityTimedOut",
14340        },
14341        Error::ChildWorkflowFailed(failure) => match failure.kind {
14342            ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14343            ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14344            ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14345        },
14346        Error::ParallelFailed(_) => "ParallelFailed",
14347        Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14348        Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14349        Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14350        _ => "RustWorkflowError",
14351    }
14352}
14353
14354fn workflow_error_non_retryable(error: &Error) -> bool {
14355    match error {
14356        Error::ActivityFailed(failure) => failure.non_retryable,
14357        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14358        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14359        Error::SagaCompensationFailed(failure) => {
14360            workflow_error_non_retryable(&failure.compensation_failure)
14361        }
14362        Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14363        _ => false,
14364    }
14365}
14366
14367fn workflow_task_integrity_error(error: &Error) -> bool {
14368    matches!(
14369        error,
14370        Error::NonDeterministicReplay(_)
14371            | Error::Protocol(_)
14372            | Error::MissingWorkflowCommandIdentity
14373            | Error::WorkflowStatePoisoned
14374    )
14375}
14376
14377fn decode_signal_event_arguments(
14378    event: &HistoryEvent,
14379    fallback_codec: &str,
14380) -> Result<Vec<AvroValue>> {
14381    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14382    validate_payload_codec(codec)?;
14383    let raw = signal_history_payload(&event.payload);
14384    let decoded = match raw.filter(|value| !value.is_null()) {
14385        Some(value) => decode_wire_avro_value(value, codec)?,
14386        None => AvroValue::Array(Vec::new()),
14387    };
14388    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14389        unreachable!("normalize_avro_arguments always returns an array");
14390    };
14391    Ok(arguments)
14392}
14393
14394fn decode_update_event_arguments(
14395    event: &HistoryEvent,
14396    fallback_codec: &str,
14397) -> Result<Vec<AvroValue>> {
14398    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14399    validate_payload_codec(codec)?;
14400    let decoded = match event
14401        .payload
14402        .get("arguments")
14403        .filter(|value| !value.is_null())
14404    {
14405        Some(value) => decode_wire_avro_value(value, codec)?,
14406        None => AvroValue::Array(Vec::new()),
14407    };
14408    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14409        unreachable!("normalize_avro_arguments always returns an array");
14410    };
14411    Ok(arguments)
14412}
14413
14414fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14415    let Some(export_events) = task
14416        .history_export
14417        .as_ref()
14418        .and_then(|export| export.get("history_events"))
14419        .and_then(Value::as_array)
14420    else {
14421        return Ok(());
14422    };
14423
14424    if export_events.len() > task.history_events.len() {
14425        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14426    }
14427
14428    Ok(())
14429}
14430
14431fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14432    let Some(export) = task.history_export.as_ref() else {
14433        return Ok(());
14434    };
14435    let signals = export
14436        .get("signals")
14437        .and_then(Value::as_array)
14438        .cloned()
14439        .unwrap_or_default();
14440    let activities = export
14441        .get("activities")
14442        .and_then(Value::as_array)
14443        .cloned()
14444        .unwrap_or_default();
14445    let export_codec = export
14446        .get("payloads")
14447        .and_then(|payloads| payloads.get("codec"))
14448        .and_then(Value::as_str)
14449        .unwrap_or(&task.payload_codec)
14450        .to_string();
14451    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14452
14453    for event in &mut task.history_events {
14454        if event.event_type == "ActivityCompleted" {
14455            let sequence = event
14456                .payload
14457                .get("sequence")
14458                .or_else(|| event.payload.get("workflow_sequence"))
14459                .and_then(value_as_u64);
14460            let Some(activity) = sequence.and_then(|sequence| {
14461                activities.iter().find(|activity| {
14462                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14463                })
14464            }) else {
14465                continue;
14466            };
14467            let Some(payload) = event.payload.as_object_mut() else {
14468                continue;
14469            };
14470            if missing_payload(payload.get("result")) {
14471                if let Some(result) = activity
14472                    .get("result")
14473                    .filter(|value| !missing_payload(Some(value)))
14474                {
14475                    payload.insert("result".to_string(), result.clone());
14476                }
14477            }
14478            for field in ["payload_codec", "activity_type"] {
14479                if payload
14480                    .get(field)
14481                    .and_then(Value::as_str)
14482                    .unwrap_or_default()
14483                    .is_empty()
14484                {
14485                    if let Some(value) = activity.get(field) {
14486                        payload.insert(field.to_string(), value.clone());
14487                    }
14488                }
14489            }
14490            continue;
14491        }
14492
14493        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14494            continue;
14495        }
14496        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14497        let command_id = event
14498            .payload
14499            .get("workflow_command_id")
14500            .or_else(|| event.raw.get("workflow_command_id"))
14501            .and_then(Value::as_str);
14502        let signal_name = event
14503            .payload
14504            .get("signal_name")
14505            .and_then(Value::as_str)
14506            .unwrap_or_default()
14507            .to_string();
14508        let matched = signals
14509            .iter()
14510            .find(|signal| {
14511                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14512            })
14513            .or_else(|| {
14514                signals.iter().find(|signal| {
14515                    command_id.is_some()
14516                        && signal.get("command_id").and_then(Value::as_str) == command_id
14517                })
14518            })
14519            .or_else(|| {
14520                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14521                let signal = signals
14522                    .iter()
14523                    .filter(|signal| {
14524                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14525                    })
14526                    .nth(*offset);
14527                if signal.is_some() {
14528                    *offset += 1;
14529                }
14530                signal
14531            });
14532        let Some(signal) = matched else {
14533            continue;
14534        };
14535        let signal_codec = signal
14536            .get("payload_codec")
14537            .and_then(Value::as_str)
14538            .unwrap_or(&export_codec);
14539        let Some(payload) = event.payload.as_object_mut() else {
14540            continue;
14541        };
14542        if missing_payload(payload.get("arguments")) {
14543            if let Some(arguments) = signal
14544                .get("arguments")
14545                .filter(|value| !missing_payload(Some(value)))
14546            {
14547                let envelope = match arguments {
14548                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14549                    other => other.clone(),
14550                };
14551                payload.insert("arguments".to_string(), envelope);
14552            }
14553        }
14554        if payload
14555            .get("payload_codec")
14556            .and_then(Value::as_str)
14557            .unwrap_or_default()
14558            .is_empty()
14559        {
14560            payload.insert("payload_codec".to_string(), json!(signal_codec));
14561        }
14562    }
14563
14564    Ok(())
14565}
14566
14567fn missing_payload(value: Option<&Value>) -> bool {
14568    match value {
14569        None | Some(Value::Null) => true,
14570        Some(Value::String(value)) => value.is_empty(),
14571        Some(_) => false,
14572    }
14573}
14574
14575fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14576    let export_signals = task
14577        .history_export
14578        .as_ref()
14579        .and_then(|export| export.get("signals"))
14580        .and_then(Value::as_array)
14581        .cloned()
14582        .unwrap_or_default();
14583    let export_codec = task
14584        .history_export
14585        .as_ref()
14586        .and_then(|export| export.get("payloads"))
14587        .and_then(|payloads| payloads.get("codec"))
14588        .and_then(Value::as_str)
14589        .unwrap_or(&task.payload_codec);
14590    let mut name_offsets: HashMap<String, usize> = HashMap::new();
14591    let mut signals = Vec::new();
14592
14593    for event in &task.history_events {
14594        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14595            continue;
14596        }
14597
14598        let name = event
14599            .payload
14600            .get("signal_name")
14601            .and_then(Value::as_str)
14602            .unwrap_or_default();
14603        if name.is_empty() {
14604            continue;
14605        }
14606        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14607        let command_id = event
14608            .payload
14609            .get("workflow_command_id")
14610            .or_else(|| event.raw.get("workflow_command_id"))
14611            .and_then(Value::as_str);
14612        let matched_export = export_signals
14613            .iter()
14614            .find(|candidate| {
14615                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14616            })
14617            .or_else(|| {
14618                export_signals.iter().find(|candidate| {
14619                    command_id.is_some()
14620                        && candidate.get("command_id").and_then(Value::as_str) == command_id
14621                })
14622            })
14623            .or_else(|| {
14624                let offset = name_offsets.entry(name.to_string()).or_default();
14625                let candidate = export_signals
14626                    .iter()
14627                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14628                    .nth(*offset);
14629                if candidate.is_some() {
14630                    *offset += 1;
14631                }
14632                candidate
14633            });
14634        let codec = event
14635            .payload
14636            .get("payload_codec")
14637            .and_then(Value::as_str)
14638            .or_else(|| {
14639                matched_export
14640                    .and_then(|signal| signal.get("payload_codec"))
14641                    .and_then(Value::as_str)
14642            })
14643            .unwrap_or(export_codec);
14644        let raw_arguments = signal_history_payload(&event.payload)
14645            .filter(|value| !value.is_null())
14646            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14647        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14648        let workflow_sequence = event
14649            .payload
14650            .get("workflow_sequence")
14651            .and_then(value_as_u64)
14652            .or_else(|| {
14653                matched_export
14654                    .and_then(|signal| signal.get("workflow_sequence"))
14655                    .and_then(value_as_u64)
14656            });
14657
14658        signals.push(QuerySignal {
14659            id: signal_id.map(str::to_string).or_else(|| {
14660                matched_export
14661                    .and_then(|signal| signal.get("id"))
14662                    .and_then(Value::as_str)
14663                    .map(str::to_string)
14664            }),
14665            name: name.to_string(),
14666            arguments,
14667            avro_arguments,
14668            workflow_sequence,
14669        });
14670    }
14671
14672    if signals.is_empty() {
14673        for signal in export_signals {
14674            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14675                continue;
14676            }
14677            let Some(name) = signal.get("name").and_then(Value::as_str) else {
14678                continue;
14679            };
14680            let codec = signal
14681                .get("payload_codec")
14682                .and_then(Value::as_str)
14683                .unwrap_or(export_codec);
14684            let (arguments, avro_arguments) =
14685                decode_query_signal_arguments(signal.get("arguments"), codec)?;
14686            signals.push(QuerySignal {
14687                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14688                name: name.to_string(),
14689                arguments,
14690                avro_arguments,
14691                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14692            });
14693        }
14694        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14695    }
14696
14697    Ok(signals)
14698}
14699
14700fn decode_query_signal_arguments(
14701    raw: Option<&Value>,
14702    codec: &str,
14703) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14704    validate_payload_codec(codec)?;
14705    let decoded = match raw.filter(|value| !value.is_null()) {
14706        Some(value) => decode_wire_avro_value(value, codec)?,
14707        None => AvroValue::Array(Vec::new()),
14708    };
14709    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14710        unreachable!("normalize_avro_arguments always returns an array");
14711    };
14712    let arguments = avro_arguments
14713        .iter()
14714        .cloned()
14715        .map(AvroValue::into_json)
14716        .collect::<Result<Vec<_>>>()?;
14717    Ok((arguments, avro_arguments))
14718}
14719
14720fn value_as_u64(value: &Value) -> Option<u64> {
14721    value
14722        .as_u64()
14723        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14724}
14725
14726#[cfg(test)]
14727mod tests {
14728    use super::*;
14729    mod runtime_payloads;
14730    use std::{
14731        fs,
14732        io::{Read, Write},
14733        net::{SocketAddr, TcpListener, TcpStream},
14734        process::Command as ProcessCommand,
14735        sync::atomic::AtomicUsize,
14736        thread,
14737    };
14738
14739    #[derive(Clone, Copy, Debug)]
14740    enum InvalidTaskPayloadCodec {
14741        Missing,
14742        Null,
14743        NonString,
14744    }
14745
14746    impl InvalidTaskPayloadCodec {
14747        fn label(self) -> &'static str {
14748            match self {
14749                Self::Missing => "missing",
14750                Self::Null => "null",
14751                Self::NonString => "non-string",
14752            }
14753        }
14754
14755        fn apply(self, task: &mut Value) {
14756            let task = task.as_object_mut().expect("task fixture object");
14757            match self {
14758                Self::Missing => {
14759                    task.remove("payload_codec");
14760                }
14761                Self::Null => {
14762                    task.insert("payload_codec".to_string(), Value::Null);
14763                }
14764                Self::NonString => {
14765                    task.insert("payload_codec".to_string(), json!(42));
14766                }
14767            }
14768        }
14769    }
14770
14771    fn fixture_envelope(value: Value) -> Value {
14772        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14773    }
14774
14775    fn fixture_blob(value: Value) -> String {
14776        encode_payload(&value, DEFAULT_CODEC)
14777            .expect("encode Avro test fixture")
14778            .blob
14779    }
14780
14781    #[test]
14782    fn client_builder_rejects_the_sdk_owned_api_suffix() {
14783        for base_url in [
14784            "http://127.0.0.1:8080/api",
14785            "http://localhost:8080/api/",
14786            "https://runtime.example.test/namespaces/orders/api",
14787        ] {
14788            let error = Client::builder(base_url)
14789                .build()
14790                .expect_err("SDK-owned /api suffix must be rejected during build");
14791
14792            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14793            assert!(
14794                error.to_string().contains("SDK appends /api automatically"),
14795                "the validation error must explain how to fix the endpoint"
14796            );
14797        }
14798    }
14799
14800    #[test]
14801    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14802        for (base_url, expected) in [
14803            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14804            (
14805                "http://localhost:8080/durable-workflow/",
14806                "http://localhost:8080/durable-workflow",
14807            ),
14808            (
14809                "https://runtime.example.test/namespaces/orders",
14810                "https://runtime.example.test/namespaces/orders",
14811            ),
14812            (
14813                "https://runtime.example.test/gateway/api/namespaces/orders",
14814                "https://runtime.example.test/gateway/api/namespaces/orders",
14815            ),
14816            (
14817                "https://api.example.test/runtime/orders/",
14818                "https://api.example.test/runtime/orders",
14819            ),
14820        ] {
14821            let client = Client::builder(base_url)
14822                .build()
14823                .expect("Server and Cloud runtime base URL must remain valid");
14824
14825            assert_eq!(client.base_url, expected);
14826        }
14827    }
14828
14829    #[test]
14830    fn workflow_completion_uses_the_additive_command_protocol_floor() {
14831        assert_eq!(
14832            workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14833            WORKER_PROTOCOL_VERSION
14834        );
14835        assert_eq!(
14836            workflow_completion_protocol_version(&[json!({
14837                "type": "upsert_search_attributes",
14838                "attributes": {"OrderStatus": "waiting"},
14839            })]),
14840            SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14841        );
14842        assert_eq!(
14843            workflow_completion_protocol_version(&[json!({
14844                "type": "upsert_search_attributes",
14845                "attributes": {"OrderStatus": "waiting"},
14846                "attribute_types": {"OrderStatus": "keyword"},
14847            })]),
14848            TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14849        );
14850        assert_eq!(
14851            workflow_completion_protocol_version(&[
14852                json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14853                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14854            ]),
14855            MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14856        );
14857        assert_eq!(
14858            workflow_completion_protocol_version(&[
14859                json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14860                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14861            ]),
14862            CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14863        );
14864        assert_eq!(
14865            workflow_completion_protocol_version(&[json!({
14866                "type": "open_condition_wait",
14867                "condition_wait_occurrence_id": "rust:condition-wait:0",
14868                "condition_key": "ready",
14869            })]),
14870            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14871        );
14872        assert_eq!(
14873            workflow_completion_protocol_version_with_message_streams(
14874                &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14875                true,
14876            ),
14877            MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14878        );
14879        assert_eq!(
14880            workflow_completion_protocol_version_with_message_streams(
14881                &[json!({
14882                    "type": "open_condition_wait",
14883                    "condition_wait_occurrence_id": "rust:condition-wait:0",
14884                    "condition_key": "ready",
14885                })],
14886                true,
14887            ),
14888            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14889        );
14890    }
14891
14892    #[test]
14893    fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14894        let manifest = portable_worker_affinity_capability_manifest();
14895
14896        for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14897            assert_eq!(manifest[capability]["supported"], json!(false));
14898            assert_eq!(
14899                manifest[capability]["minimum_protocol_version"],
14900                json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14901            );
14902            assert!(manifest[capability]["reason"]
14903                .as_str()
14904                .is_some_and(|reason| !reason.is_empty()));
14905        }
14906    }
14907
14908    fn typed_fidelity_probe() -> AvroValue {
14909        AvroValue::Map(BTreeMap::from([
14910            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
14911            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
14912            (
14913                "numeric".to_string(),
14914                AvroValue::Map(BTreeMap::from([
14915                    ("0".to_string(), AvroValue::String("zero".to_string())),
14916                    ("1".to_string(), AvroValue::String("one".to_string())),
14917                ])),
14918            ),
14919            (
14920                "nested".to_string(),
14921                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
14922                    "enabled".to_string(),
14923                    AvroValue::Boolean(true),
14924                )]))]),
14925            ),
14926            (
14927                "projection_collisions".to_string(),
14928                AvroValue::Array(projection_collision_probe()),
14929            ),
14930        ]))
14931    }
14932
14933    fn projection_collision_probe() -> Vec<AvroValue> {
14934        vec![
14935            AvroValue::Map(BTreeMap::from([
14936                ("$type".to_string(), AvroValue::String("bytes".to_string())),
14937                (
14938                    "base64".to_string(),
14939                    AvroValue::String("ordinary user text".to_string()),
14940                ),
14941            ])),
14942            AvroValue::Map(BTreeMap::from([
14943                ("$type".to_string(), AvroValue::String("map".to_string())),
14944                (
14945                    "entries".to_string(),
14946                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
14947                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
14948                        (
14949                            "value".to_string(),
14950                            AvroValue::String("user map".to_string()),
14951                        ),
14952                    ]))]),
14953                ),
14954            ])),
14955        ]
14956    }
14957
14958    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14959    struct TypedContract {
14960        nested: TypedNested,
14961        mode: TypedMode,
14962        optional: Option<String>,
14963        absent: Option<String>,
14964        items: Vec<i64>,
14965        labels: BTreeMap<String, String>,
14966        bytes: serde_bytes::ByteBuf,
14967        signed: i64,
14968        finite: f64,
14969    }
14970
14971    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14972    struct TypedNested {
14973        enabled: bool,
14974    }
14975
14976    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14977    enum TypedMode {
14978        Detailed { label: String },
14979    }
14980
14981    fn typed_contract() -> TypedContract {
14982        TypedContract {
14983            nested: TypedNested { enabled: true },
14984            mode: TypedMode::Detailed {
14985                label: "compiler-checked".to_string(),
14986            },
14987            optional: Some("present".to_string()),
14988            absent: None,
14989            items: vec![i64::MIN, 0, i64::MAX],
14990            labels: BTreeMap::from([
14991                ("language".to_string(), "rust".to_string()),
14992                ("wire".to_string(), "avro".to_string()),
14993            ]),
14994            bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
14995            signed: -9_223_372_036_854_775_000,
14996            finite: 12.5,
14997        }
14998    }
14999
15000    #[derive(Clone, Debug, Default, PartialEq)]
15001    struct ReplayCounterState {
15002        loaded: Option<String>,
15003        count: i64,
15004        finished: bool,
15005    }
15006
15007    fn replay_counter_worker() -> Worker {
15008        let client = Client::new("http://127.0.0.1:8080").expect("client");
15009        let mut worker = Worker::new(client, "rust-workers");
15010        worker.register_replayed_workflow(
15011            "replay-counter",
15012            ReplayCounterState::default,
15013            |ctx, _input, state| async move {
15014                let loaded = ctx.activity("load-counter", json!([])).await?;
15015                state.update(|current| {
15016                    current.loaded = loaded.as_str().map(str::to_string);
15017                })?;
15018                for _ in 0..2 {
15019                    let signal = ctx.wait_signal("increment").await?;
15020                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15021                    state.update(|current| current.count += amount)?;
15022                }
15023                state.update(|current| current.finished = true)?;
15024                state.read(|current| Ok(json!(current.count)))?
15025            },
15026        );
15027        worker.register_replayed_query::<ReplayCounterState, _, _>(
15028            "replay-counter",
15029            "current",
15030            |_ctx, state, _args| async move {
15031                Ok(json!({
15032                    "loaded": state.loaded,
15033                    "count": state.count,
15034                    "finished": state.finished,
15035                }))
15036            },
15037        );
15038        worker.register_replayed_query::<ReplayCounterState, _, _>(
15039            "replay-counter",
15040            "detached-mutation",
15041            |_ctx, state, _args| async move {
15042                let mut detached = (*state).clone();
15043                detached.count = 999;
15044                Ok(json!(detached.count))
15045            },
15046        );
15047        worker.register_replayed_query::<ReplayCounterState, _, _>(
15048            "replay-counter",
15049            "failed-mutation",
15050            |_ctx, state, _args| async move {
15051                let mut detached = (*state).clone();
15052                detached.count = 999;
15053                Err(Error::WorkerLoop("query refused".to_string()))
15054            },
15055        );
15056        worker
15057    }
15058
15059    fn replay_counter_query(
15060        query_name: &str,
15061        history_events: Value,
15062        run_status: &str,
15063    ) -> QueryTask {
15064        let arguments = fixture_envelope(json!([]));
15065        serde_json::from_value(json!({
15066            "query_task_id": format!("query-{query_name}"),
15067            "workflow_type": "replay-counter",
15068            "query_name": query_name,
15069            "payload_codec": DEFAULT_CODEC,
15070            "workflow_arguments": arguments.clone(),
15071            "query_arguments": arguments,
15072            "history_events": history_events,
15073            "run_status": run_status,
15074        }))
15075        .expect("query task")
15076    }
15077
15078    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15079        workflow_context_with_codec(history, DEFAULT_CODEC)
15080    }
15081
15082    fn workflow_context_with_codec(
15083        history: Vec<HistoryEvent>,
15084        payload_codec: &str,
15085    ) -> WorkflowContext {
15086        WorkflowContext {
15087            state: Arc::new(Mutex::new(
15088                WorkflowState::new_with_identity(
15089                    history,
15090                    None,
15091                    None,
15092                    "rust-workers".to_string(),
15093                    payload_codec.to_string(),
15094                    None,
15095                )
15096                .expect("valid workflow history"),
15097            )),
15098        }
15099    }
15100
15101    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15102        HistoryEvent {
15103            event_type: event_type.to_string(),
15104            payload,
15105            raw: HashMap::new(),
15106        }
15107    }
15108
15109    fn parallel_path_entry(
15110        kind: &str,
15111        base: u64,
15112        size: usize,
15113        index: usize,
15114    ) -> ParallelGroupMetadata {
15115        parallel_group_entry(base, size, index, kind)
15116    }
15117
15118    fn parallel_history_event(
15119        event_type: &str,
15120        sequence: u64,
15121        identity_field: &str,
15122        identity: &str,
15123        path: Vec<ParallelGroupMetadata>,
15124        result: Option<Value>,
15125    ) -> HistoryEvent {
15126        let mut payload = serde_json::Map::from_iter([
15127            ("sequence".to_string(), json!(sequence)),
15128            (identity_field.to_string(), json!(identity)),
15129        ]);
15130        let inner = path.last().expect("parallel history path");
15131        apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15132        payload.insert("parallel_group_path".to_string(), json!(path));
15133        if let Some(result) = result {
15134            let field = if event_type == "ChildRunCompleted" {
15135                "result"
15136            } else {
15137                "result"
15138            };
15139            payload.insert(field.to_string(), fixture_envelope(result));
15140            payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15141        }
15142        history_event(event_type, Value::Object(payload))
15143    }
15144
15145    fn nested_parallel_operations() -> Vec<ParallelOperation> {
15146        vec![
15147            ParallelOperation::activity("first", json!([])),
15148            ParallelOperation::group(vec![
15149                ParallelOperation::child_workflow(
15150                    "second",
15151                    ChildWorkflowOptions::new("child-workers"),
15152                    json!([]),
15153                ),
15154                ParallelOperation::activity("third", json!([])),
15155            ]),
15156        ]
15157    }
15158
15159    fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15160        let outer = [
15161            parallel_path_entry("mixed", 1, 3, 0),
15162            parallel_path_entry("mixed", 1, 3, 1),
15163            parallel_path_entry("mixed", 1, 3, 2),
15164        ];
15165        [
15166            vec![outer[0].clone()],
15167            vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15168            vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15169        ]
15170    }
15171
15172    #[test]
15173    fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15174        let ctx = workflow_context(Vec::new());
15175        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15176        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15177
15178        assert!(matches!(
15179            call.as_mut().poll(&mut task_context),
15180            Poll::Pending
15181        ));
15182        let commands = ctx.take_commands().expect("parallel commands");
15183        assert_eq!(
15184            commands
15185                .iter()
15186                .map(|command| command["type"].as_str().unwrap_or_default())
15187                .collect::<Vec<_>>(),
15188            [
15189                "schedule_activity",
15190                "start_child_workflow",
15191                "schedule_activity"
15192            ]
15193        );
15194        let paths = nested_parallel_paths();
15195        for (command, path) in commands.iter().zip(paths) {
15196            assert_eq!(command["parallel_group_path"], json!(path));
15197            assert_eq!(
15198                command["parallel_group_id"],
15199                json!(path.last().expect("inner group").parallel_group_id)
15200            );
15201        }
15202    }
15203
15204    fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15205        let paths = nested_parallel_paths();
15206        let third = parallel_history_event(
15207            "ActivityCompleted",
15208            3,
15209            "activity_type",
15210            "third",
15211            paths[2].clone(),
15212            Some(json!("three")),
15213        );
15214        vec![
15215            parallel_history_event(
15216                "ActivityCompleted",
15217                1,
15218                "activity_type",
15219                "first",
15220                paths[0].clone(),
15221                Some(json!("one")),
15222            ),
15223            parallel_history_event(
15224                "ChildWorkflowScheduled",
15225                2,
15226                "child_workflow_type",
15227                "second",
15228                paths[1].clone(),
15229                None,
15230            ),
15231            parallel_history_event(
15232                "ChildRunCompleted",
15233                2,
15234                "child_workflow_type",
15235                "second",
15236                paths[1].clone(),
15237                Some(json!("two")),
15238            ),
15239            third.clone(),
15240            third,
15241        ]
15242    }
15243
15244    #[test]
15245    fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15246        for _restart_or_completed_replay in 0..2 {
15247            let ctx = workflow_context(completed_nested_parallel_history());
15248            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15249            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15250            let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15251                panic!("completed nested parallel history must replay");
15252            };
15253            assert_eq!(
15254                results,
15255                vec![
15256                    ParallelResult::Activity(json!("one")),
15257                    ParallelResult::Group(vec![
15258                        ParallelResult::ChildWorkflow(ChildWorkflowResult {
15259                            parent: WorkflowIdentity {
15260                                workflow_id: None,
15261                                run_id: None,
15262                            },
15263                            child: WorkflowIdentity {
15264                                workflow_id: None,
15265                                run_id: None,
15266                            },
15267                            child_workflow_type: Some("second".to_string()),
15268                            result: json!("two"),
15269                        }),
15270                        ParallelResult::Activity(json!("three")),
15271                    ]),
15272                ]
15273            );
15274            assert!(ctx.take_commands().expect("commands").is_empty());
15275            ctx.ensure_history_consumed().expect("history consumed");
15276        }
15277    }
15278
15279    #[test]
15280    fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15281        let paths = nested_parallel_paths();
15282        let history = vec![
15283            parallel_history_event(
15284                "ActivityCompleted",
15285                1,
15286                "activity_type",
15287                "first",
15288                paths[0].clone(),
15289                Some(json!("one")),
15290            ),
15291            parallel_history_event(
15292                "ChildWorkflowScheduled",
15293                2,
15294                "child_workflow_type",
15295                "second",
15296                paths[1].clone(),
15297                None,
15298            ),
15299            parallel_history_event(
15300                "ChildRunFailed",
15301                2,
15302                "child_workflow_type",
15303                "second",
15304                paths[1].clone(),
15305                None,
15306            ),
15307            parallel_history_event(
15308                "ActivityCompleted",
15309                3,
15310                "activity_type",
15311                "third",
15312                paths[2].clone(),
15313                Some(json!("late")),
15314            ),
15315        ];
15316        let ctx = workflow_context(history);
15317        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15318        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15319        let outcome = call.as_mut().poll(&mut task_context);
15320        let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15321            panic!("one failed child must return a typed partial failure: {outcome:?}");
15322        };
15323        assert_eq!(failure.member_path, [1, 0]);
15324        assert_eq!(failure.group_id, "parallel-calls:1:3");
15325        assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15326        assert_eq!(
15327            failure
15328                .completed
15329                .iter()
15330                .map(|completion| completion.member_path.clone())
15331                .collect::<Vec<_>>(),
15332            [vec![0], vec![1, 1]]
15333        );
15334    }
15335
15336    #[test]
15337    fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15338        let paths = nested_parallel_paths();
15339        let history = vec![
15340            parallel_history_event(
15341                "ActivityScheduled",
15342                1,
15343                "activity_type",
15344                "first",
15345                paths[0].clone(),
15346                None,
15347            ),
15348            parallel_history_event(
15349                "ChildWorkflowScheduled",
15350                2,
15351                "child_workflow_type",
15352                "second",
15353                paths[1].clone(),
15354                None,
15355            ),
15356            parallel_history_event(
15357                "ActivityScheduled",
15358                3,
15359                "activity_type",
15360                "third",
15361                paths[2].clone(),
15362                None,
15363            ),
15364        ];
15365        for _restart in 0..2 {
15366            let ctx = workflow_context(history.clone());
15367            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15368            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15369            let outcome = call.as_mut().poll(&mut task_context);
15370            assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15371            assert!(ctx.take_commands().expect("commands").is_empty());
15372        }
15373    }
15374
15375    fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15376        vec![selection_group_entry(
15377            1,
15378            2,
15379            index,
15380            "activity",
15381            &SelectionMemberMetadata {
15382                key: SelectionKey::Name(key.to_string()),
15383                index,
15384                base_sequence: index as u64 + 1,
15385                size: 1,
15386                kind: "activity".to_string(),
15387            },
15388        )]
15389    }
15390
15391    fn selection_activity_event(
15392        event_type: &str,
15393        index: usize,
15394        key: &str,
15395        result: Option<Value>,
15396    ) -> HistoryEvent {
15397        let sequence = index as u64 + 1;
15398        let mut event = parallel_history_event(
15399            event_type,
15400            sequence,
15401            "activity_type",
15402            &format!("{key}-activity"),
15403            selection_path(index, key),
15404            result,
15405        );
15406        event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15407        event.raw.insert(
15408            "id".to_string(),
15409            json!(if event_type == "ActivityCompleted" {
15410                format!("event-{key}")
15411            } else {
15412                format!("{event_type}-{key}")
15413            }),
15414        );
15415        event
15416    }
15417
15418    fn selection_winner_marker() -> HistoryEvent {
15419        history_event(
15420            "SelectionResolved",
15421            json!({
15422                "selection_group_id": "select-calls:1:2",
15423                "selection_group_base_sequence": 1,
15424                "selection_group_size": 2,
15425                "member_key": "fast",
15426                "member_index": 1,
15427                "member_base_sequence": 2,
15428                "member_size": 1,
15429                "operation_kind": "activity",
15430                "operation_identity": "activity-fast",
15431                "outcome": "completed",
15432                "resolution_event_id": "event-fast",
15433                "resolution_event_type": "ActivityCompleted",
15434            }),
15435        )
15436    }
15437
15438    fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15439        ctx.select_keyed(vec![
15440            (
15441                "slow",
15442                ParallelOperation::activity_with_options(
15443                    "slow-activity",
15444                    ActivityOptions::new().task_queue("default"),
15445                    json!([]),
15446                ),
15447            ),
15448            (
15449                "fast",
15450                ParallelOperation::activity_with_options(
15451                    "fast-activity",
15452                    ActivityOptions::new().task_queue("default"),
15453                    json!([]),
15454                ),
15455            ),
15456        ])
15457    }
15458
15459    fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15460        let ctx = workflow_context(history);
15461        let mut call = Box::pin(keyed_activity_selection(&ctx));
15462        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15463        let selected = match call.as_mut().poll(&mut task_context) {
15464            Poll::Ready(Ok(selected)) => selected,
15465            Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15466            Poll::Pending => panic!("persisted selection winner must replay without pending"),
15467        };
15468        assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15469        assert_eq!(
15470            selected.value,
15471            Some(ParallelResult::Activity(json!("winner-value")))
15472        );
15473        let slow = selected
15474            .handle(&SelectionKey::Name("slow".to_string()))
15475            .expect("slow handle")
15476            .clone();
15477        let mut await_slow = Box::pin(slow.await_result());
15478        assert!(matches!(
15479            await_slow.as_mut().poll(&mut task_context),
15480            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15481        ));
15482        assert!(ctx.take_commands().expect("commands").is_empty());
15483    }
15484
15485    const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15486
15487    fn canonical_selection_history() -> Vec<HistoryEvent> {
15488        const FIXTURE: &[u8] =
15489            include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15490        assert_eq!(
15491            format!("{:x}", Sha256::digest(FIXTURE)),
15492            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15493        );
15494        let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15495
15496        serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15497    }
15498
15499    #[test]
15500    fn selection_fresh_process_entrypoint() {
15501        let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15502            return;
15503        };
15504        let persisted = fs::read(path).expect("persisted selection history");
15505        assert_eq!(
15506            format!("{:x}", Sha256::digest(&persisted)),
15507            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15508        );
15509        let fixture: Value =
15510            serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15511        let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15512            .expect("valid persisted selection history");
15513
15514        assert_persisted_selection_replay(history);
15515    }
15516
15517    #[test]
15518    fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15519        let ctx = workflow_context(Vec::new());
15520        let mut call = Box::pin(keyed_activity_selection(&ctx));
15521        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15522
15523        assert!(matches!(
15524            call.as_mut().poll(&mut task_context),
15525            Poll::Pending
15526        ));
15527        let commands = ctx.take_commands().expect("selection commands");
15528        assert_eq!(commands.len(), 2);
15529        assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15530        assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15531        assert!(commands.iter().all(|command| {
15532            command["parallel_group_id"] == json!("select-calls:1:2")
15533                && command["parallel_group_mode"] == json!("select")
15534        }));
15535    }
15536
15537    #[test]
15538    fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15539        let ctx = workflow_context(Vec::new());
15540        let mut invalid = Box::pin(ctx.select_keyed(vec![(
15541            "",
15542            ParallelOperation::activity("invalid", json!([])),
15543        )]));
15544        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15545        assert!(matches!(
15546            invalid.as_mut().poll(&mut task_context),
15547            Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15548                reason: "selection_key_invalid",
15549                ..
15550            })))
15551        ));
15552
15553        for invalid_key in [json!(""), json!(-1)] {
15554            let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15555            event.payload["selection_member_key"] = invalid_key.clone();
15556            event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15557            assert!(matches!(
15558                WorkflowState::new_with_identity(
15559                    vec![event],
15560                    None,
15561                    None,
15562                    "rust-workers".to_string(),
15563                    DEFAULT_CODEC.to_string(),
15564                    None,
15565                ),
15566                Err(Error::NonDeterministicReplay(_))
15567            ));
15568        }
15569    }
15570
15571    #[test]
15572    fn selection_preserves_valid_named_and_numeric_keys() {
15573        let ctx = workflow_context(Vec::new());
15574        let mut selection = Box::pin(ctx.select_keyed(vec![
15575            (
15576                SelectionKey::Index(0),
15577                ParallelOperation::activity("numeric", json!([])),
15578            ),
15579            (
15580                SelectionKey::Name("named".to_string()),
15581                ParallelOperation::timer(Duration::from_secs(1)),
15582            ),
15583        ]));
15584        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15585
15586        assert!(matches!(
15587            selection.as_mut().poll(&mut task_context),
15588            Poll::Pending
15589        ));
15590        let commands = ctx.take_commands().expect("selection commands");
15591        assert_eq!(commands[0]["selection_member_key"], json!(0));
15592        assert_eq!(commands[1]["selection_member_key"], json!("named"));
15593    }
15594
15595    #[test]
15596    fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15597        let history = canonical_selection_history();
15598        assert_persisted_selection_replay(history.clone());
15599
15600        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15601            .join("tests/fixtures/durable_selection_runtime_history.json");
15602        let output =
15603            ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15604                .args([
15605                    "--exact",
15606                    "tests::selection_fresh_process_entrypoint",
15607                    "--nocapture",
15608                ])
15609                .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15610                .output()
15611                .expect("run fresh selection replay process");
15612
15613        assert!(
15614            output.status.success(),
15615            "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15616            String::from_utf8_lossy(&output.stdout),
15617            String::from_utf8_lossy(&output.stderr),
15618        );
15619    }
15620
15621    #[test]
15622    fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15623        let mut history = canonical_selection_history();
15624        history.retain(|event| event.event_type != "SelectionResolved");
15625        let ctx = workflow_context(history);
15626        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15627        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15628
15629        assert!(matches!(
15630            selection.as_mut().poll(&mut task_context),
15631            Poll::Pending
15632        ));
15633        assert!(ctx.take_commands().expect("commands").is_empty());
15634        assert!(
15635            ctx.matched_recorded_pending()
15636                .expect("selection pending state"),
15637            "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15638        );
15639    }
15640
15641    #[test]
15642    fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15643        for (terminal_event, predicate_satisfied, timeout_seconds) in [
15644            ("ConditionWaitSatisfied", true, None),
15645            ("ConditionWaitTimedOut", false, Some(0)),
15646        ] {
15647            let member = SelectionMemberMetadata {
15648                key: SelectionKey::Name("condition".to_string()),
15649                index: 0,
15650                base_sequence: 1,
15651                size: 1,
15652                kind: "condition".to_string(),
15653            };
15654            let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15655            let mut payload = json!({
15656                "sequence": 1,
15657                "condition_wait_id": "condition-1",
15658                "condition_wait_occurrence_id": "rust:condition-wait:0",
15659                "condition_key": "ready",
15660                "condition_definition_fingerprint": "sha256:ready-v1",
15661                "parallel_group_path": path,
15662            });
15663            payload
15664                .as_object_mut()
15665                .expect("condition history payload")
15666                .extend(
15667                    serde_json::to_value(&path[0])
15668                        .expect("condition selection metadata")
15669                        .as_object()
15670                        .expect("condition selection metadata object")
15671                        .clone(),
15672                );
15673            if let Some(timeout_seconds) = timeout_seconds {
15674                payload["timeout_seconds"] = json!(timeout_seconds);
15675            }
15676            let history = vec![
15677                history_event("ConditionWaitOpened", payload.clone()),
15678                history_event(terminal_event, payload),
15679            ];
15680            let ctx = workflow_context(history);
15681            let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15682            if timeout_seconds.is_some() {
15683                options = options.timeout(Duration::ZERO);
15684            }
15685            let mut selection = Box::pin(ctx.select_keyed(vec![(
15686                "condition",
15687                ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15688            )]));
15689            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15690
15691            assert!(matches!(
15692                selection.as_mut().poll(&mut task_context),
15693                Poll::Pending
15694            ));
15695            assert!(ctx.take_commands().expect("commands").is_empty());
15696            assert!(
15697                ctx.matched_recorded_pending()
15698                    .expect("condition selection pending state"),
15699                "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15700            );
15701        }
15702    }
15703
15704    #[test]
15705    fn selection_immediate_condition_members_open_a_durable_wait() {
15706        for predicate_satisfied in [true, false] {
15707            let ctx = workflow_context(Vec::new());
15708            let mut selection = Box::pin(ctx.select_keyed(vec![(
15709                "condition",
15710                ParallelOperation::condition(
15711                    ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15712                    move || Ok(predicate_satisfied),
15713                ),
15714            )]));
15715            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15716
15717            assert!(matches!(
15718                selection.as_mut().poll(&mut task_context),
15719                Poll::Pending
15720            ));
15721            let commands = ctx.take_commands().expect("condition selection command");
15722            assert_eq!(commands.len(), 1);
15723            assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15724            assert_eq!(commands[0]["timeout_seconds"], json!(0));
15725            assert_eq!(
15726                commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15727                json!("select")
15728            );
15729        }
15730    }
15731
15732    #[test]
15733    fn selection_loser_cancellation_is_explicit_and_idempotent() {
15734        let history = vec![
15735            selection_activity_event("ActivityScheduled", 0, "slow", None),
15736            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15737            selection_winner_marker(),
15738        ];
15739        let ctx = workflow_context(history.clone());
15740        let mut call = Box::pin(keyed_activity_selection(&ctx));
15741        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15742        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15743            panic!("winner must replay");
15744        };
15745        let slow = selected
15746            .handle(&SelectionKey::Name("slow".to_string()))
15747            .expect("slow handle")
15748            .clone();
15749        let mut cancel = Box::pin(slow.cancel());
15750        assert!(matches!(
15751            cancel.as_mut().poll(&mut task_context),
15752            Poll::Pending
15753        ));
15754        assert!(matches!(
15755            cancel.as_mut().poll(&mut task_context),
15756            Poll::Pending
15757        ));
15758        let commands = ctx.take_commands().expect("cancel command");
15759        assert_eq!(commands.len(), 1);
15760        assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15761        assert_eq!(commands[0]["member_key"], json!("slow"));
15762
15763        let mut cancelled_history = history;
15764        cancelled_history.push(history_event(
15765            "SelectionOperationCancelled",
15766            json!({
15767                "selection_group_id": "select-calls:1:2",
15768                "member_key": "slow",
15769                "member_index": 0,
15770                "member_base_sequence": 1,
15771                "member_size": 1,
15772                "operation_kind": "activity",
15773                "operation_identity": "activity-slow",
15774                "cancelled_at": "2026-08-27T00:00:00Z",
15775            }),
15776        ));
15777        let replayed = workflow_context(cancelled_history);
15778        let mut call = Box::pin(keyed_activity_selection(&replayed));
15779        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15780            panic!("winner must replay after cancellation");
15781        };
15782        let slow = selected
15783            .handle(&SelectionKey::Name("slow".to_string()))
15784            .expect("slow handle")
15785            .clone();
15786        let mut cancel = Box::pin(slow.cancel());
15787        assert!(matches!(
15788            cancel.as_mut().poll(&mut task_context),
15789            Poll::Ready(Ok(()))
15790        ));
15791        assert!(replayed.take_commands().expect("commands").is_empty());
15792    }
15793
15794    #[test]
15795    fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15796        let base_history = vec![
15797            selection_activity_event("ActivityScheduled", 0, "slow", None),
15798            selection_activity_event("ActivityScheduled", 1, "fast", None),
15799            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15800            selection_winner_marker(),
15801        ];
15802        for (field, corrupt) in [
15803            ("member_key", json!("fast")),
15804            ("member_index", json!(1)),
15805            ("member_base_sequence", json!(3)),
15806            ("member_size", json!(2)),
15807            ("operation_kind", json!("timer")),
15808            ("operation_identity", json!("forged")),
15809        ] {
15810            let mut cancellation = json!({
15811                "selection_group_id": "select-calls:1:2",
15812                "member_key": "slow",
15813                "member_index": 0,
15814                "member_base_sequence": 1,
15815                "member_size": 1,
15816                "operation_kind": "activity",
15817                "operation_identity": "activity-slow",
15818            });
15819            cancellation[field] = corrupt;
15820            let mut history = base_history.clone();
15821            history.push(history_event("SelectionOperationCancelled", cancellation));
15822            let ctx = workflow_context(history);
15823            let mut selection = Box::pin(keyed_activity_selection(&ctx));
15824            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15825
15826            assert!(matches!(
15827                selection.as_mut().poll(&mut task_context),
15828                Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15829            ));
15830        }
15831    }
15832
15833    #[test]
15834    fn selection_child_identity_prefers_the_durable_run_id() {
15835        let ctx = workflow_context(vec![history_event(
15836            "ChildWorkflowScheduled",
15837            json!({
15838                "sequence": 1,
15839                "child_workflow_type": "child",
15840                "child_workflow_instance_id": "child-instance",
15841                "child_workflow_run_id": "child-run",
15842            }),
15843        )]);
15844        let state = ctx.state.lock().expect("workflow state");
15845
15846        assert_eq!(
15847            selection_operation_identity(&state, "child", 1, 1),
15848            "child-run"
15849        );
15850    }
15851
15852    #[test]
15853    fn selection_activity_identity_requires_canonical_execution_id() {
15854        let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15855        let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15856        let mut fast_completed =
15857            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15858        for event in [&mut fast_open, &mut fast_completed] {
15859            event
15860                .payload
15861                .as_object_mut()
15862                .expect("activity payload")
15863                .remove("activity_execution_id");
15864            event.payload["activity_id"] = json!("forged-activity-id");
15865        }
15866        let mut marker = selection_winner_marker();
15867        marker.payload["operation_identity"] = json!("forged-activity-id");
15868        let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15869        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15870        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15871
15872        assert!(matches!(
15873            selection.as_mut().poll(&mut task_context),
15874            Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15875        ));
15876    }
15877
15878    #[test]
15879    fn selection_completion_before_cancellation_remains_awaitable() {
15880        let history = vec![
15881            selection_activity_event("ActivityScheduled", 0, "slow", None),
15882            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15883            selection_winner_marker(),
15884            selection_activity_event(
15885                "ActivityCompleted",
15886                0,
15887                "slow",
15888                Some(json!("completed-first")),
15889            ),
15890        ];
15891        let ctx = workflow_context(history);
15892        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15893        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15894        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15895            panic!("winner must replay");
15896        };
15897        let slow = selected
15898            .handle(&SelectionKey::Name("slow".to_string()))
15899            .expect("slow handle")
15900            .clone();
15901        let mut cancel = Box::pin(slow.cancel());
15902        assert!(matches!(
15903            cancel.as_mut().poll(&mut task_context),
15904            Poll::Ready(Ok(()))
15905        ));
15906        let mut await_slow = Box::pin(slow.await_result());
15907        assert!(matches!(
15908            await_slow.as_mut().poll(&mut task_context),
15909            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
15910        ));
15911        let commands = ctx.take_commands().expect("commands");
15912        assert!(commands.is_empty());
15913    }
15914
15915    #[test]
15916    fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
15917        let nested_member = SelectionMemberMetadata {
15918            key: SelectionKey::Name("nested".to_string()),
15919            index: 0,
15920            base_sequence: 1,
15921            size: 2,
15922            kind: "group".to_string(),
15923        };
15924        let deadline_member = SelectionMemberMetadata {
15925            key: SelectionKey::Name("deadline".to_string()),
15926            index: 1,
15927            base_sequence: 3,
15928            size: 1,
15929            kind: "timer".to_string(),
15930        };
15931        let nested_paths = [
15932            vec![
15933                selection_group_entry(1, 3, 0, "mixed", &nested_member),
15934                parallel_group_entry(1, 2, 0, "activity"),
15935            ],
15936            vec![
15937                selection_group_entry(1, 3, 1, "mixed", &nested_member),
15938                parallel_group_entry(1, 2, 1, "activity"),
15939            ],
15940        ];
15941        let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
15942        let mut timer_fired = parallel_history_event(
15943            "TimerFired",
15944            3,
15945            "timer_id",
15946            "timer-3",
15947            deadline_path.clone(),
15948            None,
15949        );
15950        timer_fired.payload["delay_seconds"] = json!(0);
15951        timer_fired
15952            .raw
15953            .insert("id".to_string(), json!("timer-fired"));
15954        let mut timer_scheduled = parallel_history_event(
15955            "TimerScheduled",
15956            3,
15957            "timer_id",
15958            "timer-3",
15959            deadline_path,
15960            None,
15961        );
15962        timer_scheduled.payload["delay_seconds"] = json!(0);
15963        let history = vec![
15964            parallel_history_event(
15965                "ActivityScheduled",
15966                1,
15967                "activity_type",
15968                "nested-first",
15969                nested_paths[0].clone(),
15970                None,
15971            ),
15972            parallel_history_event(
15973                "ActivityScheduled",
15974                2,
15975                "activity_type",
15976                "nested-second",
15977                nested_paths[1].clone(),
15978                None,
15979            ),
15980            timer_scheduled,
15981            timer_fired,
15982            history_event(
15983                "SelectionResolved",
15984                json!({
15985                    "selection_group_id": "select-calls:1:3",
15986                    "selection_group_base_sequence": 1,
15987                    "selection_group_size": 3,
15988                    "member_key": "deadline",
15989                    "member_index": 1,
15990                    "member_base_sequence": 3,
15991                    "member_size": 1,
15992                    "operation_kind": "timer",
15993                    "operation_identity": "timer-3",
15994                    "outcome": "completed",
15995                    "resolution_event_id": "timer-fired",
15996                    "resolution_event_type": "TimerFired",
15997                }),
15998            ),
15999            parallel_history_event(
16000                "ActivityFailed",
16001                2,
16002                "activity_type",
16003                "nested-second",
16004                nested_paths[1].clone(),
16005                None,
16006            ),
16007        ];
16008        let ctx = workflow_context(history);
16009        let mut selection = Box::pin(ctx.select_keyed(vec![
16010            (
16011                "nested",
16012                ParallelOperation::group(vec![
16013                    ParallelOperation::activity("nested-first", json!([])),
16014                    ParallelOperation::activity("nested-second", json!([])),
16015                ]),
16016            ),
16017            ("deadline", ParallelOperation::timer(Duration::ZERO)),
16018        ]));
16019        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16020        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16021            panic!("deadline winner must replay");
16022        };
16023        let nested = selected
16024            .handle(&SelectionKey::Name("nested".to_string()))
16025            .expect("nested handle")
16026            .clone();
16027        let mut cancel = Box::pin(nested.cancel());
16028        assert!(matches!(
16029            cancel.as_mut().poll(&mut task_context),
16030            Poll::Ready(Ok(()))
16031        ));
16032        let mut await_nested = Box::pin(nested.await_result());
16033
16034        assert!(matches!(
16035            await_nested.as_mut().poll(&mut task_context),
16036            Poll::Ready(Err(Error::ActivityFailed(_)))
16037        ));
16038        assert!(ctx.take_commands().expect("commands").is_empty());
16039    }
16040
16041    #[test]
16042    fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16043        let ctx = workflow_context(Vec::new());
16044        let mut call = Box::pin(ctx.select(vec![
16045            ParallelOperation::child_workflow(
16046                "child",
16047                ChildWorkflowOptions::new("children"),
16048                json!([]),
16049            ),
16050            ParallelOperation::timer(Duration::from_secs(30)),
16051            ParallelOperation::signal("approval"),
16052            ParallelOperation::condition(
16053                ConditionWaitOptions::new("ready", "sha256:ready"),
16054                || Ok(false),
16055            ),
16056            ParallelOperation::group(vec![
16057                ParallelOperation::activity("nested-one", json!([])),
16058                ParallelOperation::activity("nested-two", json!([])),
16059            ]),
16060        ]));
16061        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16062        assert!(matches!(
16063            call.as_mut().poll(&mut task_context),
16064            Poll::Pending
16065        ));
16066        let commands = ctx.take_commands().expect("selection commands");
16067        assert_eq!(
16068            commands
16069                .iter()
16070                .map(|command| command["type"].as_str().unwrap_or_default())
16071                .collect::<Vec<_>>(),
16072            [
16073                "start_child_workflow",
16074                "start_timer",
16075                "open_signal_wait",
16076                "open_condition_wait",
16077                "schedule_activity",
16078                "schedule_activity",
16079            ]
16080        );
16081        assert!(commands.iter().all(|command| {
16082            command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16083        }));
16084        assert_eq!(
16085            commands[4]["parallel_group_path"].as_array().map(Vec::len),
16086            Some(2)
16087        );
16088        assert_eq!(
16089            commands[4]["parallel_group_path"][0]["selection_member_kind"],
16090            json!("group")
16091        );
16092        assert_eq!(
16093            commands[5]["parallel_group_path"][0]["selection_member_kind"],
16094            json!("group")
16095        );
16096
16097        let one_leaf_ctx = workflow_context(Vec::new());
16098        let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16099            ParallelOperation::activity("nested-only", json!([])),
16100        ])]));
16101        assert!(matches!(
16102            one_leaf.as_mut().poll(&mut task_context),
16103            Poll::Pending
16104        ));
16105        let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16106        assert_eq!(one_leaf_commands.len(), 1);
16107        assert_eq!(
16108            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16109            json!("group")
16110        );
16111        assert_eq!(
16112            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16113            json!(1)
16114        );
16115    }
16116
16117    async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16118        let mut saga = ctx.saga();
16119        let outcome = async {
16120            let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16121            saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16122            let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16123            saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16124            ctx.activity("trip.charge", json!([])).await?;
16125            Ok(json!({"status": "booked"}))
16126        }
16127        .await;
16128        saga.finish(outcome).await
16129    }
16130
16131    fn saga_activity(
16132        event_type: &str,
16133        sequence: u64,
16134        activity_type: &str,
16135        result: Option<Value>,
16136    ) -> HistoryEvent {
16137        let mut payload = json!({
16138            "sequence": sequence,
16139            "activity_type": activity_type,
16140            "message": format!("{activity_type} failed"),
16141            "exception_type": "PlannedFailure",
16142            "non_retryable": true,
16143        });
16144        if let Some(result) = result {
16145            payload["result"] = fixture_envelope(result);
16146        }
16147        history_event(event_type, payload)
16148    }
16149
16150    #[test]
16151    fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16152        let completed_hotel_compensation = saga_activity(
16153            "ActivityCompleted",
16154            4,
16155            "trip.cancel-hotel",
16156            Some(Value::Null),
16157        );
16158        let history = vec![
16159            saga_activity(
16160                "ActivityCompleted",
16161                1,
16162                "trip.reserve-flight",
16163                Some(json!("flight-1")),
16164            ),
16165            saga_activity(
16166                "ActivityCompleted",
16167                2,
16168                "trip.reserve-hotel",
16169                Some(json!("hotel-1")),
16170            ),
16171            saga_activity("ActivityFailed", 3, "trip.charge", None),
16172            completed_hotel_compensation.clone(),
16173            completed_hotel_compensation,
16174        ];
16175
16176        for _restart in 0..2 {
16177            let ctx = workflow_context(history.clone());
16178            let mut future = Box::pin(trip_saga(ctx.clone()));
16179            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16180            assert!(matches!(
16181                future.as_mut().poll(&mut task_context),
16182                Poll::Pending
16183            ));
16184            let commands = ctx.take_commands().expect("compensation command");
16185            assert_eq!(commands.len(), 1);
16186            assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16187        }
16188    }
16189
16190    #[test]
16191    fn saga_compensation_failure_preserves_both_typed_failures() {
16192        let history = vec![
16193            saga_activity(
16194                "ActivityCompleted",
16195                1,
16196                "trip.reserve-flight",
16197                Some(json!("flight-1")),
16198            ),
16199            saga_activity(
16200                "ActivityCompleted",
16201                2,
16202                "trip.reserve-hotel",
16203                Some(json!("hotel-1")),
16204            ),
16205            saga_activity("ActivityFailed", 3, "trip.charge", None),
16206            saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16207        ];
16208        let ctx = workflow_context(history);
16209        let mut future = Box::pin(trip_saga(ctx));
16210        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16211        let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16212            future.as_mut().poll(&mut task_context)
16213        else {
16214            panic!("compensation failure must remain structured");
16215        };
16216        assert!(matches!(
16217            *failure.initiating_failure,
16218            Error::ActivityFailed(_)
16219        ));
16220        assert!(matches!(
16221            *failure.compensation_failure,
16222            Error::ActivityFailed(_)
16223        ));
16224        assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16225        assert_eq!(failure.compensation_registration_order, 2);
16226    }
16227
16228    #[test]
16229    fn saga_compensates_cooperative_cancellation() {
16230        let ctx = workflow_context(vec![saga_activity(
16231            "ActivityCompleted",
16232            1,
16233            "trip.reserve-flight",
16234            Some(json!("flight-1")),
16235        )]);
16236        ctx.state.lock().expect("state").cancel_requested = true;
16237        let run = {
16238            let ctx = ctx.clone();
16239            async move {
16240                let mut saga = ctx.saga();
16241                let outcome = async {
16242                    let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16243                    saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16244                    ctx.throw_if_cancellation_requested()?;
16245                    Ok(json!("unexpected"))
16246                }
16247                .await;
16248                saga.finish(outcome).await
16249            }
16250        };
16251        let mut future = Box::pin(run);
16252        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16253        assert!(matches!(
16254            future.as_mut().poll(&mut task_context),
16255            Poll::Pending
16256        ));
16257        let commands = ctx.take_commands().expect("cancellation compensation");
16258        assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16259    }
16260
16261    fn workflow_task(
16262        workflow_type: &str,
16263        history_events: Vec<HistoryEvent>,
16264        payload_codec: &str,
16265    ) -> WorkflowTask {
16266        WorkflowTask {
16267            task_id: format!("wft-{workflow_type}"),
16268            workflow_command_id: None,
16269            workflow_id: Some(format!("wf-{workflow_type}")),
16270            run_id: Some(format!("run-{workflow_type}")),
16271            workflow_type: workflow_type.to_string(),
16272            cancel_requested: false,
16273            payload_codec: payload_codec.to_string(),
16274            arguments: Some(
16275                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16276            ),
16277            total_history_events: Some(history_events.len() as u64),
16278            history_size_bytes: None,
16279            continue_as_new_recommended: None,
16280            history_budget_pressure: None,
16281            history_events,
16282            next_history_page_token: None,
16283            workflow_task_attempt: 1,
16284            workflow_signal_id: None,
16285            signal_name: None,
16286            signal_arguments: None,
16287            workflow_update_id: None,
16288            update_name: None,
16289            lease_owner: Some("rust-worker".to_string()),
16290        }
16291    }
16292
16293    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16294    struct SideEffectProbe {
16295        request_id: String,
16296        attempt: u32,
16297    }
16298
16299    #[test]
16300    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16301        let calls = AtomicUsize::new(0);
16302        let ctx = workflow_context(Vec::new());
16303        let value = ctx
16304            .side_effect(|| {
16305                calls.fetch_add(1, Ordering::SeqCst);
16306                SideEffectProbe {
16307                    request_id: "request-42".to_string(),
16308                    attempt: 3,
16309                }
16310            })
16311            .expect("first side effect");
16312        assert_eq!(value.attempt, 3);
16313        assert_eq!(calls.load(Ordering::SeqCst), 1);
16314        let commands = ctx.take_commands().expect("commands");
16315        assert_eq!(commands.len(), 1);
16316        assert_eq!(commands[0]["type"], "record_side_effect");
16317        assert_eq!(
16318            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16319            serde_json::to_value(&value).expect("value")
16320        );
16321
16322        let replay = workflow_context(vec![history_event(
16323            "SideEffectRecorded",
16324            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16325        )]);
16326        let replayed: SideEffectProbe = replay
16327            .side_effect(|| {
16328                calls.fetch_add(1, Ordering::SeqCst);
16329                panic!("committed side-effect callbacks must not run during replay")
16330            })
16331            .expect("replayed side effect");
16332        assert_eq!(replayed, value);
16333        assert_eq!(calls.load(Ordering::SeqCst), 1);
16334        assert!(replay.take_commands().expect("commands").is_empty());
16335        replay.ensure_history_consumed().expect("history consumed");
16336    }
16337
16338    #[test]
16339    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16340        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16341        let value = ctx
16342            .side_effect(|| SideEffectProbe {
16343                request_id: "avro-request".to_string(),
16344                attempt: 1,
16345            })
16346            .expect("Avro side effect");
16347        let uuid = ctx.uuid_v4().expect("deterministic UUID");
16348        let commands = ctx.take_commands().expect("commands");
16349        assert_eq!(commands.len(), 2);
16350        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16351        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16352        assert_eq!(
16353            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16354            serde_json::to_value(&value).expect("value")
16355        );
16356
16357        let replay = workflow_context_with_codec(
16358            vec![
16359                history_event(
16360                    "SideEffectRecorded",
16361                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16362                ),
16363                history_event(
16364                    "SideEffectRecorded",
16365                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16366                ),
16367            ],
16368            DEFAULT_CODEC,
16369        );
16370        let replayed: SideEffectProbe = replay
16371            .side_effect(|| panic!("Avro callback must not run"))
16372            .expect("replayed Avro value");
16373        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16374        assert_eq!(replayed, value);
16375        assert_eq!(replayed_uuid, uuid);
16376        assert!(replay.take_commands().expect("commands").is_empty());
16377    }
16378
16379    #[test]
16380    fn typed_side_effect_replay_preserves_bytes_and_maps() {
16381        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16382        let value = ctx
16383            .side_effect_avro_value(typed_fidelity_probe)
16384            .expect("typed side effect");
16385        let commands = ctx.take_commands().expect("side-effect command");
16386        assert_eq!(
16387            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16388                .expect("recorded side effect"),
16389            value
16390        );
16391
16392        let replay = workflow_context_with_codec(
16393            vec![history_event(
16394                "SideEffectRecorded",
16395                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16396            )],
16397            DEFAULT_CODEC,
16398        );
16399        assert_eq!(
16400            replay
16401                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16402                .expect("replayed typed side effect"),
16403            value
16404        );
16405    }
16406
16407    #[test]
16408    fn ordered_side_effects_share_the_durable_command_stream() {
16409        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16410        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16411        let ctx = workflow_context(vec![
16412            history_event(
16413                "SideEffectRecorded",
16414                json!({"sequence": 1, "result": first}),
16415            ),
16416            history_event(
16417                "SideEffectRecorded",
16418                json!({"sequence": 2, "result": second}),
16419            ),
16420        ]);
16421        let first: String = ctx
16422            .side_effect(|| panic!("first callback must not run"))
16423            .expect("first replay");
16424        let second: i32 = ctx
16425            .side_effect(|| panic!("second callback must not run"))
16426            .expect("second replay");
16427        assert_eq!(first, "first");
16428        assert_eq!(second, 29);
16429        ctx.ensure_history_consumed().expect("ordered history");
16430
16431        let reordered = workflow_context(vec![history_event(
16432            "VersionMarkerRecorded",
16433            json!({
16434                "sequence": 1,
16435                "change_id": "before-side-effect",
16436                "version": 1,
16437                "min_supported": 1,
16438                "max_supported": 1,
16439            }),
16440        )]);
16441        let error = reordered
16442            .side_effect(|| "new".to_string())
16443            .expect_err("command reordering must fail");
16444        assert!(matches!(
16445            error,
16446            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16447                if reason == "recorded_command_mismatch"
16448        ));
16449    }
16450
16451    #[test]
16452    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16453        let ctx = workflow_context(Vec::new());
16454        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16455        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16456        assert!(ctx.patched("new-search").expect("patch"));
16457        ctx.deprecate_patch("new-search").expect("deprecate patch");
16458        let commands = ctx.take_commands().expect("commands");
16459        assert_eq!(commands.len(), 2);
16460        assert_eq!(commands[0]["type"], "record_version_marker");
16461        assert_eq!(commands[0]["version"], 2);
16462        assert_eq!(commands[1]["change_id"], "new-search");
16463
16464        let replay = workflow_context(vec![history_event(
16465            "VersionMarkerRecorded",
16466            json!({
16467                "sequence": 1,
16468                "change_id": "checkout-v2",
16469                "version": 2,
16470                "min_supported": 1,
16471                "max_supported": 2,
16472            }),
16473        )]);
16474        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16475        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16476        assert!(replay.take_commands().expect("commands").is_empty());
16477        replay.ensure_history_consumed().expect("history consumed");
16478    }
16479
16480    #[test]
16481    fn version_markers_reject_incompatible_or_malformed_history() {
16482        let incompatible = workflow_context(vec![history_event(
16483            "VersionMarkerRecorded",
16484            json!({
16485                "sequence": 1,
16486                "change_id": "checkout-v2",
16487                "version": 1,
16488                "min_supported": 1,
16489                "max_supported": 2,
16490            }),
16491        )]);
16492        let error = incompatible
16493            .get_version("checkout-v2", 2, 3)
16494            .expect_err("old version is unsupported");
16495        assert!(matches!(
16496            error,
16497            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16498                if reason == "version_marker_incompatible_range"
16499        ));
16500
16501        for (history, reason) in [
16502            (
16503                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16504                "side_effect_result_missing",
16505            ),
16506            (
16507                vec![history_event(
16508                    "SideEffectRecorded",
16509                    json!({
16510                        "sequence": 1,
16511                        "result": {"codec": "avro", "blob": "not-base64"},
16512                    }),
16513                )],
16514                "side_effect_payload_incompatible",
16515            ),
16516            (
16517                vec![history_event(
16518                    "SideEffectRecorded",
16519                    json!({"sequence": 1, "result": {"unwrapped": true}}),
16520                )],
16521                "side_effect_payload_malformed",
16522            ),
16523            (
16524                vec![history_event(
16525                    "VersionMarkerRecorded",
16526                    json!({
16527                        "sequence": 1,
16528                        "change_id": "change",
16529                        "version": 1,
16530                        "min_supported": 2,
16531                        "max_supported": 1,
16532                    }),
16533                )],
16534                "version_marker_history_range_invalid",
16535            ),
16536        ] {
16537            let error = WorkflowState::new(
16538                history,
16539                "rust-workers".to_string(),
16540                DEFAULT_CODEC.to_string(),
16541                None,
16542            )
16543            .expect_err("malformed history must fail");
16544            assert!(matches!(
16545                error,
16546                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16547                    if actual == reason
16548            ));
16549        }
16550    }
16551
16552    #[test]
16553    fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16554        let history = vec![history_event(
16555            "SearchAttributesUpserted",
16556            json!({
16557                "sequence": 1,
16558                "attributes": {"customer_tier": "gold"},
16559                "attribute_types": {"customer_tier": "keyword"},
16560                "merged": {"customer_tier": "gold"}
16561            }),
16562        )];
16563
16564        let matching = workflow_context(history.clone());
16565        matching
16566            .upsert_search_attributes(
16567                SearchAttributeUpdate::new()
16568                    .keyword("customer_tier", "gold")
16569                    .expect("keyword update"),
16570            )
16571            .expect("matching typed update must replay");
16572        matching
16573            .ensure_history_consumed()
16574            .expect("history consumed");
16575
16576        let changed_type = workflow_context(history.clone());
16577        let error = changed_type
16578            .upsert_search_attributes(
16579                SearchAttributeUpdate::new()
16580                    .string("customer_tier", "gold")
16581                    .expect("string update"),
16582            )
16583            .expect_err("same JSON value with a different declaration must be nondeterministic");
16584        let Error::NonDeterministicReplay(failure) = error else {
16585            panic!("typed identity drift must be a replay failure");
16586        };
16587        assert_eq!(failure.reason, "search_attribute_type_mismatch");
16588        assert_eq!(failure.sequence, Some(1));
16589
16590        let changed_value = workflow_context(history);
16591        let error = changed_value
16592            .upsert_search_attributes(
16593                SearchAttributeUpdate::new()
16594                    .keyword("customer_tier", "platinum")
16595                    .expect("keyword update"),
16596            )
16597            .expect_err("changed values must be nondeterministic");
16598        let Error::NonDeterministicReplay(failure) = error else {
16599            panic!("value drift must be a replay failure");
16600        };
16601        assert_eq!(failure.reason, "search_attribute_value_mismatch");
16602    }
16603
16604    #[test]
16605    fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16606        let history = vec![history_event(
16607            "SearchAttributesUpserted",
16608            json!({
16609                "sequence": 1,
16610                "attributes": {"customer_tier": "gold"},
16611                "merged": {"customer_tier": "gold"}
16612            }),
16613        )];
16614
16615        for update in [
16616            SearchAttributeUpdate::new()
16617                .keyword("customer_tier", "gold")
16618                .expect("keyword update"),
16619            SearchAttributeUpdate::new()
16620                .string("customer_tier", "gold")
16621                .expect("string update"),
16622        ] {
16623            let restarted = workflow_context(history.clone());
16624            restarted
16625                .upsert_search_attributes(update)
16626                .expect("legacy history constrains values but has unknown type identity");
16627            restarted
16628                .ensure_history_consumed()
16629                .expect("history consumed");
16630        }
16631    }
16632
16633    #[test]
16634    fn search_attribute_command_emits_canonical_types() {
16635        let ctx = workflow_context(Vec::new());
16636        ctx.upsert_search_attributes(
16637            SearchAttributeUpdate::new()
16638                .keyword("customer_tier", "gold")
16639                .expect("keyword update")
16640                .int("attempts", 3)
16641                .expect("int update")
16642                .delete("obsolete")
16643                .expect("delete update"),
16644        )
16645        .expect("valid search attributes");
16646
16647        assert_eq!(
16648            ctx.take_commands().expect("commands"),
16649            vec![json!({
16650                "type": "upsert_search_attributes",
16651                "attributes": {
16652                    "attempts": 3,
16653                    "customer_tier": "gold",
16654                    "obsolete": null
16655                },
16656                "attribute_types": {
16657                    "attempts": "int",
16658                    "customer_tier": "keyword"
16659                }
16660            })]
16661        );
16662    }
16663
16664    #[test]
16665    fn duplicate_side_effects_and_version_markers_are_rejected() {
16666        let duplicate_side_effect = WorkflowState::new(
16667            vec![
16668                history_event(
16669                    "SideEffectRecorded",
16670                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16671                ),
16672                history_event(
16673                    "SideEffectRecorded",
16674                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16675                ),
16676            ],
16677            "rust-workers".to_string(),
16678            DEFAULT_CODEC.to_string(),
16679            None,
16680        )
16681        .expect_err("duplicate side effect");
16682        assert!(matches!(
16683            duplicate_side_effect,
16684            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16685                if reason == "duplicate_side_effect_record"
16686        ));
16687
16688        let marker = |sequence| {
16689            history_event(
16690                "VersionMarkerRecorded",
16691                json!({
16692                    "sequence": sequence,
16693                    "change_id": "same-change",
16694                    "version": 1,
16695                    "min_supported": 1,
16696                    "max_supported": 1,
16697                }),
16698            )
16699        };
16700        let duplicate_marker = WorkflowState::new(
16701            vec![marker(1), marker(3)],
16702            "rust-workers".to_string(),
16703            DEFAULT_CODEC.to_string(),
16704            None,
16705        )
16706        .expect_err("duplicate marker");
16707        assert!(matches!(
16708            duplicate_marker,
16709            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16710                if reason == "duplicate_version_marker"
16711        ));
16712    }
16713
16714    #[test]
16715    fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16716        let mut state = WorkflowState::new(
16717            Vec::new(),
16718            "rust-workers".to_string(),
16719            DEFAULT_CODEC.to_string(),
16720            None,
16721        )
16722        .expect("workflow state");
16723        state.workflow_command_identity = "command-7".to_string();
16724        let context = WorkflowContext {
16725            state: Arc::new(Mutex::new(state)),
16726        };
16727        let item =
16728            WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16729
16730        context
16731            .append_workflow_stream("output", &[item], Some(10))
16732            .expect("append command");
16733        context
16734            .error_workflow_stream("output", "producer failed", None)
16735            .expect("error command");
16736        let commands = context.take_commands().expect("commands");
16737
16738        assert_eq!(commands[0]["type"], "record_side_effect");
16739        assert_eq!(
16740            commands[0]["workflow_stream"]["command_identity"],
16741            "command-7"
16742        );
16743        assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16744        assert_eq!(
16745            commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16746            "dw-stream:command-7:0:0"
16747        );
16748        assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16749
16750        let recorded = history_event(
16751            "SideEffectRecorded",
16752            json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16753        );
16754        let mut replay_state = WorkflowState::new(
16755            vec![recorded],
16756            "rust-workers".to_string(),
16757            DEFAULT_CODEC.to_string(),
16758            None,
16759        )
16760        .expect("replay state");
16761        replay_state.workflow_command_identity = "command-7".to_string();
16762        let replay_context = WorkflowContext {
16763            state: Arc::new(Mutex::new(replay_state)),
16764        };
16765        replay_context
16766            .append_workflow_stream(
16767                "output",
16768                &[WorkflowStreamAppendItem::from_reference(
16769                    "s3://bucket/item.avro",
16770                )],
16771                Some(10),
16772            )
16773            .expect("replayed append");
16774        assert!(replay_context
16775            .take_commands()
16776            .expect("replayed commands")
16777            .is_empty());
16778    }
16779
16780    #[test]
16781    fn workflow_stream_authoring_requires_server_durable_command_identity() {
16782        let context = workflow_context(Vec::new());
16783        let error = context
16784            .append_workflow_stream(
16785                "output",
16786                &[WorkflowStreamAppendItem::from_reference(
16787                    "s3://bucket/item.avro",
16788                )],
16789                None,
16790            )
16791            .expect_err("stream append without durable command identity must fail closed");
16792
16793        assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16794        assert!(context.take_commands().expect("commands").is_empty());
16795    }
16796
16797    #[test]
16798    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16799        fn worker(calls: Arc<AtomicUsize>) -> Worker {
16800            let client = Client::new("http://127.0.0.1:8080").expect("client");
16801            let mut worker = Worker::new(client, "rust-workers");
16802            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16803                let calls = Arc::clone(&calls);
16804                async move {
16805                    let captured = ctx.side_effect(|| {
16806                        calls.fetch_add(1, Ordering::SeqCst);
16807                        "captured-once".to_string()
16808                    })?;
16809                    let version = ctx.get_version("cold-restart", 1, 2)?;
16810                    Ok(json!({"captured": captured, "version": version}))
16811                }
16812            });
16813            worker
16814        }
16815
16816        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16817            WorkflowTask {
16818                task_id: "wft-side-effect-version".to_string(),
16819                workflow_command_id: None,
16820                workflow_id: Some("wf-side-effect-version".to_string()),
16821                run_id: Some("run-side-effect-version".to_string()),
16822                workflow_type: "rust.side-effect-version".to_string(),
16823                cancel_requested: false,
16824                payload_codec: DEFAULT_CODEC.to_string(),
16825                arguments: Some(
16826                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16827                ),
16828                history_events,
16829                total_history_events: None,
16830                history_size_bytes: None,
16831                continue_as_new_recommended: None,
16832                history_budget_pressure: None,
16833                next_history_page_token: None,
16834                workflow_task_attempt: 1,
16835                workflow_signal_id: None,
16836                signal_name: None,
16837                signal_arguments: None,
16838                workflow_update_id: None,
16839                update_name: None,
16840                lease_owner: Some("rust-worker".to_string()),
16841            }
16842        }
16843
16844        let calls = Arc::new(AtomicUsize::new(0));
16845        let initial = worker(Arc::clone(&calls))
16846            .execute_workflow_task(task(Vec::new()))
16847            .expect("initial execution");
16848        assert_eq!(
16849            initial
16850                .iter()
16851                .map(|command| &command["type"])
16852                .collect::<Vec<_>>(),
16853            vec![
16854                "record_side_effect",
16855                "record_version_marker",
16856                "complete_workflow"
16857            ]
16858        );
16859        assert_eq!(calls.load(Ordering::SeqCst), 1);
16860
16861        let restarted = worker(Arc::clone(&calls));
16862        let replayed = restarted
16863            .execute_workflow_task(task(vec![
16864                history_event(
16865                    "SideEffectRecorded",
16866                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16867                ),
16868                history_event(
16869                    "VersionMarkerRecorded",
16870                    json!({
16871                        "sequence": 2,
16872                        "change_id": "cold-restart",
16873                        "version": 2,
16874                        "min_supported": 1,
16875                        "max_supported": 2,
16876                    }),
16877                ),
16878            ]))
16879            .expect("cold replay");
16880        assert_eq!(replayed.len(), 1);
16881        assert_eq!(replayed[0]["type"], "complete_workflow");
16882        assert_eq!(calls.load(Ordering::SeqCst), 1);
16883    }
16884
16885    #[test]
16886    fn side_effect_replay_rejects_changed_rust_value_type() {
16887        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16888        let ctx = workflow_context(vec![history_event(
16889            "SideEffectRecorded",
16890            json!({"sequence": 1, "result": result}),
16891        )]);
16892        let error = ctx
16893            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16894            .expect_err("changed type must fail replay");
16895        assert!(matches!(
16896            error,
16897            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16898                if reason == "side_effect_type_mismatch"
16899        ));
16900    }
16901
16902    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16903        vec![
16904            history_event(
16905                "ActivityScheduled",
16906                json!({
16907                    "sequence": 1,
16908                    "activity_type": "flaky",
16909                    "activity_execution_id": "act-1",
16910                    "activity": {
16911                        "id": "act-1",
16912                        "sequence": 1,
16913                        "type": "flaky",
16914                        "queue": "critical-activities",
16915                        "execution_mode": null,
16916                        "retry_policy": {
16917                            "snapshot_version": 1,
16918                            "max_attempts": 3,
16919                            "backoff_seconds": [2, 4],
16920                            "start_to_close_timeout": 30,
16921                            "schedule_to_start_timeout": 5,
16922                            "schedule_to_close_timeout": 90,
16923                            "heartbeat_timeout": 10,
16924                            "non_retryable_error_types": ["PermanentError"]
16925                        }
16926                    }
16927                }),
16928            ),
16929            history_event(
16930                "ActivityStarted",
16931                json!({
16932                    "sequence": 1,
16933                    "activity_type": "flaky",
16934                    "activity_execution_id": "act-1",
16935                    "activity_attempt_id": "attempt-1",
16936                    "attempt_number": 1
16937                }),
16938            ),
16939            history_event(
16940                "ActivityRetryScheduled",
16941                json!({
16942                    "sequence": 1,
16943                    "activity_type": "flaky",
16944                    "activity_execution_id": "act-1",
16945                    "activity_attempt_id": "attempt-1",
16946                    "attempt_number": 1,
16947                    "retry_after_attempt": 1,
16948                    "retry_backoff_seconds": 2,
16949                    "failure_category": "activity",
16950                    "exception_type": "TransientError"
16951                }),
16952            ),
16953            history_event(
16954                "ActivityStarted",
16955                json!({
16956                    "sequence": 1,
16957                    "activity_type": "flaky",
16958                    "activity_execution_id": "act-1",
16959                    "activity_attempt_id": "attempt-2",
16960                    "attempt_number": 2
16961                }),
16962            ),
16963            history_event(
16964                "ActivityCompleted",
16965                json!({
16966                    "sequence": 1,
16967                    "activity_type": "flaky",
16968                    "activity_execution_id": "act-1",
16969                    "activity_attempt_id": "attempt-2",
16970                    "attempt_number": 2,
16971                    "payload_codec": DEFAULT_CODEC,
16972                    "result": fixture_envelope(json!({"status":"recovered"}))
16973                }),
16974            ),
16975        ]
16976    }
16977
16978    fn retry_activity_options() -> ActivityOptions {
16979        ActivityOptions::new()
16980            .task_queue("critical-activities")
16981            .retry_policy(
16982                ActivityRetryPolicy::new(3)
16983                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
16984                    .non_retryable_error_type("PermanentError"),
16985            )
16986            .start_to_close_timeout(Duration::from_secs(30))
16987            .schedule_to_start_timeout(Duration::from_secs(5))
16988            .schedule_to_close_timeout(Duration::from_secs(90))
16989            .heartbeat_timeout(Duration::from_secs(10))
16990    }
16991
16992    #[test]
16993    fn fixed_avro_value_round_trips_json_values() {
16994        let value = json!({"greeting": "hello", "count": 3, "ok": true});
16995        let envelope = PayloadEnvelope::avro(&value).expect("encode");
16996        assert_eq!(envelope.codec, DEFAULT_CODEC);
16997        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
16998    }
16999
17000    #[tokio::test]
17001    async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
17002        let client = Client::new("http://127.0.0.1:8080").expect("client");
17003        let mut worker = Worker::new(client, "rust-workers");
17004        worker.register_typed_workflow(
17005            "typed.contract.workflow",
17006            |_ctx, input: TypedContract| async move { Ok(input) },
17007        );
17008        worker.register_typed_activity(
17009            "typed.contract.activity",
17010            |_ctx, input: TypedContract| async move { Ok(input) },
17011        );
17012
17013        let expected = typed_contract();
17014        let arguments = AvroValue::Array(vec![
17015            AvroValue::from_serialize(&expected).expect("typed request")
17016        ]);
17017        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
17018        let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
17019        workflow.arguments = Some(envelope.clone());
17020        let commands = worker
17021            .execute_workflow_task(workflow)
17022            .expect("typed workflow task");
17023        let workflow_result: TypedContract =
17024            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17025                .expect("workflow result envelope")
17026                .deserialize()
17027                .expect("workflow result type");
17028        assert_eq!(workflow_result, expected);
17029
17030        let activity = ActivityTask {
17031            task_id: "typed-contract-activity".to_string(),
17032            activity_attempt_id: Some("typed-contract-attempt".to_string()),
17033            attempt_id: None,
17034            activity_type: "typed.contract.activity".to_string(),
17035            payload_codec: DEFAULT_CODEC.to_string(),
17036            arguments: Some(envelope),
17037            attempt_number: 1,
17038            lease_owner: Some("rust-worker".to_string()),
17039        };
17040        let activity_result: TypedContract = worker
17041            .execute_activity_task(activity)
17042            .await
17043            .expect("typed activity task")
17044            .deserialize()
17045            .expect("activity result type");
17046        assert_eq!(activity_result, expected);
17047    }
17048
17049    #[tokio::test]
17050    async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17051        let client = Client::new("http://127.0.0.1:8080").expect("client");
17052        let mut worker = Worker::new(client, "rust-workers");
17053        worker.register_typed_workflow(
17054            "typed.shape.workflow",
17055            |_ctx, input: TypedContract| async move { Ok(input) },
17056        );
17057        worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17058            Ok(f64::NAN)
17059        });
17060
17061        let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17062        workflow.arguments = Some(
17063            encode_typed_envelope(
17064                &AvroValue::Array(vec![
17065                    AvroValue::String("first".to_string()),
17066                    AvroValue::String("second".to_string()),
17067                ]),
17068                DEFAULT_CODEC,
17069            )
17070            .expect("malformed typed arguments"),
17071        );
17072        let commands = worker
17073            .execute_workflow_task(workflow)
17074            .expect("shape mismatch becomes a workflow failure");
17075        let message = commands[0]["message"].as_str().expect("failure message");
17076        assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17077        assert!(message.contains(type_name::<TypedContract>()));
17078        assert!(message.contains("task carried 2 arguments"));
17079
17080        let activity = ActivityTask {
17081            task_id: "typed-unsupported-activity".to_string(),
17082            activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17083            attempt_id: None,
17084            activity_type: "typed.unsupported.activity".to_string(),
17085            payload_codec: DEFAULT_CODEC.to_string(),
17086            arguments: Some(
17087                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17088                    .expect("unit arguments"),
17089            ),
17090            attempt_number: 1,
17091            lease_owner: Some("rust-worker".to_string()),
17092        };
17093        let Error::HandlerType {
17094            handler_kind,
17095            handler_name,
17096            value_kind,
17097            rust_type,
17098            message,
17099        } = worker
17100            .execute_activity_task(activity)
17101            .await
17102            .expect_err("non-finite handler output must fail")
17103        else {
17104            panic!("expected contextual handler type failure");
17105        };
17106        assert_eq!(handler_kind, HandlerKind::Activity);
17107        assert_eq!(handler_name, "typed.unsupported.activity");
17108        assert_eq!(value_kind, HandlerValueKind::Result);
17109        assert_eq!(rust_type, type_name::<f64>());
17110        assert!(message.contains("non_finite_float"));
17111    }
17112
17113    #[tokio::test]
17114    async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17115        #[derive(Clone, Default)]
17116        struct State {
17117            observed: Option<TypedContract>,
17118        }
17119
17120        let client = Client::new("http://127.0.0.1:8080").expect("client");
17121        let mut worker = Worker::new(client, "rust-workers");
17122        worker.register_typed_replayed_workflow(
17123            "typed.contract.replayed",
17124            State::default,
17125            |ctx, input: TypedContract, state| async move {
17126                let result: TypedContract =
17127                    ctx.activity_typed("typed.contract.activity", input).await?;
17128                state.update(|current| current.observed = Some(result.clone()))?;
17129                Ok(result)
17130            },
17131        );
17132        worker.register_replayed_query::<State, _, _>(
17133            "typed.contract.replayed",
17134            "observed",
17135            |_ctx, state, _args| async move {
17136                Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17137            },
17138        );
17139
17140        let expected = typed_contract();
17141        let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17142        let workflow_arguments =
17143            encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17144                .expect("workflow arguments");
17145        let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17146        let task = QueryTask {
17147            query_task_id: "typed-replay-query".to_string(),
17148            query_task_attempt: 1,
17149            lease_owner: Some("rust-worker".to_string()),
17150            workflow_id: Some("typed-replay".to_string()),
17151            run_id: Some("typed-replay-run".to_string()),
17152            workflow_type: "typed.contract.replayed".to_string(),
17153            query_name: "observed".to_string(),
17154            payload_codec: DEFAULT_CODEC.to_string(),
17155            workflow_arguments: Some(workflow_arguments),
17156            query_arguments: Some(
17157                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17158                    .expect("query arguments"),
17159            ),
17160            history_events: vec![
17161                history_event(
17162                    "ActivityScheduled",
17163                    json!({
17164                        "sequence": 1,
17165                        "activity_type": "typed.contract.activity"
17166                    }),
17167                ),
17168                history_event(
17169                    "ActivityCompleted",
17170                    json!({
17171                        "sequence": 1,
17172                        "activity_type": "typed.contract.activity",
17173                        "payload_codec": DEFAULT_CODEC,
17174                        "result": result
17175                    }),
17176                ),
17177            ],
17178            history_export: None,
17179            run_status: Some("completed".to_string()),
17180        };
17181
17182        assert_eq!(
17183            worker
17184                .execute_query_task(task)
17185                .await
17186                .expect("typed replay query")
17187                .deserialize::<i64>()
17188                .expect("query result"),
17189            expected.signed
17190        );
17191    }
17192
17193    #[tokio::test]
17194    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17195        let client = Client::new("http://127.0.0.1:8080").expect("client");
17196        let mut worker = Worker::new(client, "rust-workers");
17197        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17198        worker
17199            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17200        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17201            Ok(input)
17202        });
17203        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17204            Ok(input)
17205        });
17206        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17207            Ok(AvroValue::Array(
17208                ctx.wait_signal_avro_value("changed").await?,
17209            ))
17210        });
17211
17212        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17213        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17214
17215        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17216        workflow.arguments = Some(envelope.clone());
17217        let commands = worker
17218            .execute_workflow_task(workflow)
17219            .expect("typed workflow task");
17220        assert_eq!(commands[0]["type"], "complete_workflow");
17221        assert_eq!(
17222            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17223                .expect("typed workflow result"),
17224            arguments
17225        );
17226
17227        let activity = ActivityTask {
17228            task_id: "activity-typed".to_string(),
17229            activity_attempt_id: Some("attempt-typed".to_string()),
17230            attempt_id: None,
17231            activity_type: "typed.activity".to_string(),
17232            payload_codec: DEFAULT_CODEC.to_string(),
17233            arguments: Some(envelope.clone()),
17234            attempt_number: 1,
17235            lease_owner: Some("rust-worker".to_string()),
17236        };
17237        assert_eq!(
17238            worker
17239                .execute_activity_task(activity)
17240                .await
17241                .expect("typed activity result"),
17242            arguments
17243        );
17244
17245        let query = QueryTask {
17246            query_task_id: "query-typed".to_string(),
17247            query_task_attempt: 1,
17248            lease_owner: Some("rust-worker".to_string()),
17249            workflow_id: Some("typed-1".to_string()),
17250            run_id: Some("run-typed".to_string()),
17251            workflow_type: "typed.echo".to_string(),
17252            query_name: "inspect".to_string(),
17253            payload_codec: DEFAULT_CODEC.to_string(),
17254            workflow_arguments: Some(
17255                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17256                    .expect("workflow input"),
17257            ),
17258            query_arguments: Some(envelope.clone()),
17259            history_events: Vec::new(),
17260            history_export: None,
17261            run_status: Some("running".to_string()),
17262        };
17263        assert_eq!(
17264            worker
17265                .execute_query_task(query)
17266                .await
17267                .expect("typed query result"),
17268            arguments
17269        );
17270
17271        let mut update = workflow_task(
17272            "typed.echo",
17273            vec![history_event(
17274                "UpdateAccepted",
17275                json!({
17276                    "update_id": "update-typed",
17277                    "update_name": "replace",
17278                    "arguments": envelope.clone(),
17279                }),
17280            )],
17281            DEFAULT_CODEC,
17282        );
17283        update.workflow_update_id = Some("update-typed".to_string());
17284        update.update_name = Some("replace".to_string());
17285        let commands = worker
17286            .execute_workflow_task(update)
17287            .expect("typed update task");
17288        assert_eq!(commands[0]["type"], "complete_update");
17289        assert_eq!(
17290            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17291                .expect("typed update result"),
17292            arguments
17293        );
17294
17295        let mut signal = workflow_task(
17296            "typed.signal",
17297            vec![history_event(
17298                "SignalReceived",
17299                json!({
17300                    "signal_id": "signal-typed",
17301                    "signal_name": "changed",
17302                    "arguments": envelope.clone(),
17303                }),
17304            )],
17305            DEFAULT_CODEC,
17306        );
17307        signal.workflow_signal_id = Some("signal-typed".to_string());
17308        signal.signal_name = Some("changed".to_string());
17309        signal.signal_arguments = Some(envelope);
17310        let commands = worker
17311            .execute_workflow_task(signal)
17312            .expect("typed signal resume");
17313        assert_eq!(
17314            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17315                .expect("typed signal result"),
17316            arguments
17317        );
17318    }
17319
17320    #[tokio::test]
17321    async fn typed_helpers_never_parse_json_inspection_projection() {
17322        let collision_values = projection_collision_probe();
17323        let expected = AvroValue::Array(collision_values.clone());
17324        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17325
17326        let activity_context = workflow_context_with_codec(
17327            vec![history_event(
17328                "ActivityCompleted",
17329                json!({
17330                    "sequence": 1,
17331                    "activity_type": "collision.activity",
17332                    "payload_codec": DEFAULT_CODEC,
17333                    "result": envelope.clone(),
17334                }),
17335            )],
17336            DEFAULT_CODEC,
17337        );
17338        assert_eq!(
17339            activity_context
17340                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17341                .await
17342                .expect("typed activity collision result"),
17343            expected
17344        );
17345
17346        let signal_context = workflow_context_with_codec(
17347            vec![
17348                history_event(
17349                    "SignalWaitOpened",
17350                    json!({"sequence": 1, "signal_name": "collision"}),
17351                ),
17352                history_event(
17353                    "SignalApplied",
17354                    json!({
17355                        "sequence": 1,
17356                        "signal_name": "collision",
17357                        "payload_codec": DEFAULT_CODEC,
17358                        "value": envelope.clone(),
17359                    }),
17360                ),
17361            ],
17362            DEFAULT_CODEC,
17363        );
17364        assert_eq!(
17365            signal_context
17366                .wait_signal_avro_value("collision")
17367                .await
17368                .expect("typed signal collision arguments"),
17369            collision_values
17370        );
17371
17372        let child_context = workflow_context_with_codec(
17373            vec![
17374                history_event(
17375                    "ChildWorkflowScheduled",
17376                    json!({
17377                        "sequence": 1,
17378                        "child_workflow_instance_id": "collision-child",
17379                        "child_workflow_run_id": "collision-run",
17380                        "child_workflow_type": "collision.child",
17381                    }),
17382                ),
17383                history_event(
17384                    "ChildRunCompleted",
17385                    json!({
17386                        "sequence": 1,
17387                        "child_workflow_instance_id": "collision-child",
17388                        "child_workflow_run_id": "collision-run",
17389                        "child_workflow_type": "collision.child",
17390                        "payload_codec": DEFAULT_CODEC,
17391                        "result": envelope,
17392                    }),
17393                ),
17394            ],
17395            DEFAULT_CODEC,
17396        );
17397        let child = child_context
17398            .start_child_workflow_avro_value(
17399                "collision.child",
17400                ChildWorkflowOptions::new("collision-workers"),
17401                AvroValue::Array(Vec::new()),
17402            )
17403            .await
17404            .expect("typed child collision result");
17405        assert_eq!(child.result, expected);
17406    }
17407
17408    #[tokio::test]
17409    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17410        let client = Client::new("http://127.0.0.1:8080").expect("client");
17411        let mut worker = Worker::new(client, "rust-workers");
17412        worker.register_replayed_workflow_avro_value(
17413            "typed.replayed",
17414            || (),
17415            |_ctx, input, _state| async move { Ok(input) },
17416        );
17417        worker.register_replayed_query_avro_value::<(), _, _>(
17418            "typed.replayed",
17419            "inspect",
17420            |ctx, _state, args| async move {
17421                let mut signals = ctx.signals_avro_value("collision");
17422                let signal = signals
17423                    .pop()
17424                    .map(AvroValue::Array)
17425                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17426                Ok(AvroValue::Array(vec![
17427                    ctx.workflow_input_avro_value().clone(),
17428                    signal,
17429                    args,
17430                ]))
17431            },
17432        );
17433        let arguments = AvroValue::Array(projection_collision_probe());
17434        let signal_arguments =
17435            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17436        let task = QueryTask {
17437            query_task_id: "query-typed-replay".to_string(),
17438            query_task_attempt: 1,
17439            lease_owner: Some("rust-worker".to_string()),
17440            workflow_id: Some("typed-replay".to_string()),
17441            run_id: Some("run-typed-replay".to_string()),
17442            workflow_type: "typed.replayed".to_string(),
17443            query_name: "inspect".to_string(),
17444            payload_codec: DEFAULT_CODEC.to_string(),
17445            workflow_arguments: Some(
17446                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17447            ),
17448            query_arguments: Some(
17449                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17450            ),
17451            history_events: vec![history_event(
17452                "SignalReceived",
17453                json!({
17454                    "signal_id": "collision-signal",
17455                    "signal_name": "collision",
17456                    "workflow_sequence": 1,
17457                    "payload_codec": DEFAULT_CODEC,
17458                    "arguments": signal_arguments,
17459                }),
17460            )],
17461            history_export: None,
17462            run_status: Some("completed".to_string()),
17463        };
17464
17465        assert_eq!(
17466            worker
17467                .execute_query_task(task)
17468                .await
17469                .expect("typed replay query"),
17470            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17471        );
17472    }
17473
17474    #[test]
17475    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17476        let value = BTreeMap::from([(1_i32, "integer key")]);
17477        let error = PayloadEnvelope::avro(&value)
17478            .expect_err("integer map keys must fail")
17479            .to_string();
17480
17481        assert!(error.contains("invalid_map_key"));
17482    }
17483
17484    #[test]
17485    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17486        let envelope = PayloadEnvelope {
17487            codec: "json".to_string(),
17488            blob: r#"{"greeting":"hello"}"#.to_string(),
17489        };
17490
17491        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17492        let diagnostic = error.to_string();
17493        assert!(diagnostic.contains("unsupported_payload_codec"));
17494        assert!(diagnostic.contains("codec=\"avro\""));
17495        assert!(diagnostic.contains("HTTP document transport"));
17496    }
17497
17498    #[test]
17499    fn untagged_json_payload_value_fails_closed() {
17500        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17501            .expect_err("untagged JSON payload values must fail");
17502        let diagnostic = error.to_string();
17503        assert!(diagnostic.contains("unsupported_payload_codec"));
17504        assert!(diagnostic.contains("untagged durable payload"));
17505        assert!(diagnostic.contains("HTTP document transport"));
17506    }
17507
17508    #[test]
17509    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17510        let envelope = PayloadEnvelope {
17511            codec: DEFAULT_CODEC.to_string(),
17512            blob: BASE64.encode([0x01]),
17513        };
17514
17515        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17516        assert!(error.to_string().contains("invalid_payload_framing"));
17517    }
17518
17519    #[tokio::test]
17520    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17521        let server = MockWorkerServer::start();
17522        let client = Client::builder(server.base_url())
17523            .timeout(Duration::from_secs(2))
17524            .build()
17525            .expect("client");
17526        let invalid_commands = [
17527            json!({
17528                "type": "complete_workflow",
17529                "result": {"codec": "json", "blob": null}
17530            }),
17531            json!({
17532                "type": "schedule_activity",
17533                "arguments": {"codec": "yaml", "blob": "ignored"}
17534            }),
17535            json!({
17536                "type": "start_child_workflow",
17537                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17538            }),
17539            json!({"type": "continue_as_new", "arguments": []}),
17540            json!({"type": "complete_update"}),
17541            json!({"type": "record_side_effect", "result": null}),
17542            json!({
17543                "type": "start_service_operation",
17544                "payload_codec": DEFAULT_CODEC,
17545                "request_payload": "raw-avro-bytes"
17546            }),
17547        ];
17548
17549        for command in invalid_commands {
17550            let error = client
17551                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17552                .await
17553                .expect_err("invalid durable payload must fail locally");
17554            let diagnostic = error.to_string();
17555            assert!(
17556                diagnostic.contains("unsupported_payload_codec")
17557                    || diagnostic.contains("invalid_payload_envelope")
17558                    || diagnostic.contains("untagged durable payload"),
17559                "unexpected validation diagnostic: {diagnostic}"
17560            );
17561        }
17562
17563        assert_eq!(
17564            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17565            0,
17566            "invalid command payloads must not reach HTTP transport"
17567        );
17568    }
17569
17570    #[test]
17571    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17572        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17573        let commands = [
17574            json!({"type": "complete_workflow", "result": envelope.clone()}),
17575            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17576            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17577            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17578            json!({"type": "complete_update", "result": envelope.clone()}),
17579            json!({"type": "record_side_effect", "result": envelope.clone()}),
17580            json!({
17581                "type": "start_service_operation",
17582                "payload_codec": DEFAULT_CODEC,
17583                "request_payload": envelope.clone()
17584            }),
17585            json!({
17586                "type": "complete_workflow",
17587                "result": envelope,
17588                "metadata": {
17589                    "codec": "json",
17590                    "payload_codec": "customer-codec",
17591                    "result": {"codec": "yaml", "blob": null}
17592                }
17593            }),
17594        ];
17595
17596        validate_workflow_task_commands(&commands)
17597            .expect("customer metadata must not become a protocol codec declaration");
17598    }
17599
17600    #[test]
17601    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17602        assert_eq!(
17603            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17604            AvroValue::Array(Vec::new())
17605        );
17606        assert_eq!(
17607            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17608            AvroValue::Array(Vec::new())
17609        );
17610
17611        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17612        signal.signal_name = Some("empty-signal".to_string());
17613        signal.signal_arguments = None;
17614        let decoded = decode_resume_signal(&signal)
17615            .expect("valid Avro signal")
17616            .expect("named signal resumes the workflow");
17617        assert!(decoded.arguments.is_empty());
17618    }
17619
17620    #[tokio::test]
17621    async fn malformed_task_level_codecs_become_pre_handler_failures() {
17622        let client = Client::new("http://127.0.0.1:8080").expect("client");
17623        let mut worker = Worker::new(client, "rust-workers");
17624        let handler_calls = Arc::new(AtomicUsize::new(0));
17625
17626        let calls = Arc::clone(&handler_calls);
17627        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17628            calls.fetch_add(1, Ordering::SeqCst);
17629            async move { Ok(Value::Null) }
17630        });
17631        let calls = Arc::clone(&handler_calls);
17632        worker.register_activity("codec.activity", move |_ctx, _args| {
17633            calls.fetch_add(1, Ordering::SeqCst);
17634            async move { Ok(Value::Null) }
17635        });
17636        let calls = Arc::clone(&handler_calls);
17637        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17638            calls.fetch_add(1, Ordering::SeqCst);
17639            async move { Ok(Value::Null) }
17640        });
17641
17642        let mut failures = Vec::new();
17643        for codec_case in [
17644            InvalidTaskPayloadCodec::Missing,
17645            InvalidTaskPayloadCodec::Null,
17646            InvalidTaskPayloadCodec::NonString,
17647        ] {
17648            let mut workflow = json!({
17649                "task_id": format!("workflow-{}", codec_case.label()),
17650                "workflow_type": "codec.workflow"
17651            });
17652            codec_case.apply(&mut workflow);
17653            match serde_json::from_value::<WorkflowTask>(workflow) {
17654                Ok(task) => match worker.execute_workflow_task(task) {
17655                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17656                    outcome => failures.push(format!(
17657                        "workflow {} codec returned {outcome:?}",
17658                        codec_case.label()
17659                    )),
17660                },
17661                Err(error) => failures.push(format!(
17662                    "workflow {} codec failed transport deserialization: {error}",
17663                    codec_case.label()
17664                )),
17665            }
17666
17667            let mut activity = json!({
17668                "task_id": format!("activity-{}", codec_case.label()),
17669                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17670                "activity_type": "codec.activity",
17671                "attempt_number": 1
17672            });
17673            codec_case.apply(&mut activity);
17674            match serde_json::from_value::<ActivityTask>(activity) {
17675                Ok(task) => match worker.execute_activity_task(task).await {
17676                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17677                    outcome => failures.push(format!(
17678                        "activity {} codec returned {outcome:?}",
17679                        codec_case.label()
17680                    )),
17681                },
17682                Err(error) => failures.push(format!(
17683                    "activity {} codec failed transport deserialization: {error}",
17684                    codec_case.label()
17685                )),
17686            }
17687
17688            let mut query = json!({
17689                "query_task_id": format!("query-{}", codec_case.label()),
17690                "workflow_type": "codec.workflow",
17691                "query_name": "known"
17692            });
17693            codec_case.apply(&mut query);
17694            match serde_json::from_value::<QueryTask>(query) {
17695                Ok(task) => match worker.execute_query_task(task).await {
17696                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17697                    outcome => failures.push(format!(
17698                        "query {} codec returned {outcome:?}",
17699                        codec_case.label()
17700                    )),
17701                },
17702                Err(error) => failures.push(format!(
17703                    "query {} codec failed transport deserialization: {error}",
17704                    codec_case.label()
17705                )),
17706            }
17707        }
17708
17709        assert!(failures.is_empty(), "{}", failures.join("\n"));
17710        assert_eq!(
17711            handler_calls.load(Ordering::SeqCst),
17712            0,
17713            "invalid task codecs must not invoke a handler"
17714        );
17715    }
17716
17717    #[tokio::test]
17718    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17719        for codec_case in [
17720            InvalidTaskPayloadCodec::Missing,
17721            InvalidTaskPayloadCodec::Null,
17722            InvalidTaskPayloadCodec::NonString,
17723        ] {
17724            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17725            let client = Client::builder(server.base_url())
17726                .timeout(Duration::from_secs(2))
17727                .build()
17728                .expect("client");
17729            let mut worker = Worker::new(client, "rust-workers")
17730                .worker_id("codec-worker")
17731                .poll_timeout(Duration::from_millis(10));
17732            let handler_calls = Arc::new(AtomicUsize::new(0));
17733
17734            let calls = Arc::clone(&handler_calls);
17735            worker.register_workflow("codec.workflow", move |_ctx, _args| {
17736                calls.fetch_add(1, Ordering::SeqCst);
17737                async move { Ok(Value::Null) }
17738            });
17739            let calls = Arc::clone(&handler_calls);
17740            worker.register_activity("codec.activity", move |_ctx, _args| {
17741                calls.fetch_add(1, Ordering::SeqCst);
17742                async move { Ok(Value::Null) }
17743            });
17744            let calls = Arc::clone(&handler_calls);
17745            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17746                calls.fetch_add(1, Ordering::SeqCst);
17747                async move { Ok(Value::Null) }
17748            });
17749
17750            assert_eq!(
17751                worker.run_once().await.expect("invalid tasks are settled"),
17752                3,
17753                "all {} codec tasks must be handled",
17754                codec_case.label()
17755            );
17756            assert_eq!(
17757                handler_calls.load(Ordering::SeqCst),
17758                0,
17759                "{} task codecs must fail before every handler",
17760                codec_case.label()
17761            );
17762
17763            for path in [
17764                "/api/worker/workflow-tasks/codec-workflow/fail",
17765                "/api/worker/activity-tasks/codec-activity/fail",
17766                "/api/worker/query-tasks/codec-query/fail",
17767            ] {
17768                let body = server.request_body(path);
17769                assert!(
17770                    body["failure"]["message"]
17771                        .as_str()
17772                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
17773                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
17774                    codec_case.label()
17775                );
17776            }
17777            assert_eq!(
17778                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17779                    ["reason"],
17780                "query_payload_decode_failed"
17781            );
17782            for path in [
17783                "/api/worker/workflow-tasks/codec-workflow/complete",
17784                "/api/worker/activity-tasks/codec-activity/complete",
17785                "/api/worker/query-tasks/codec-query/complete",
17786            ] {
17787                assert_eq!(
17788                    server.request_count(path),
17789                    0,
17790                    "invalid {} codec task reached {path}",
17791                    codec_case.label()
17792                );
17793            }
17794        }
17795    }
17796
17797    #[tokio::test]
17798    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17799        let client = Client::new("http://127.0.0.1:8080").expect("client");
17800        let mut worker = Worker::new(client, "rust-workers");
17801        let handler_calls = Arc::new(AtomicUsize::new(0));
17802
17803        let calls = Arc::clone(&handler_calls);
17804        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17805            calls.fetch_add(1, Ordering::SeqCst);
17806            async move { Ok(Value::Null) }
17807        });
17808        let calls = Arc::clone(&handler_calls);
17809        worker.register_activity("codec.activity", move |_ctx, _args| {
17810            calls.fetch_add(1, Ordering::SeqCst);
17811            async move { Ok(Value::Null) }
17812        });
17813        let calls = Arc::clone(&handler_calls);
17814        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17815            calls.fetch_add(1, Ordering::SeqCst);
17816            async move { Ok(Value::Null) }
17817        });
17818        let calls = Arc::clone(&handler_calls);
17819        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17820            calls.fetch_add(1, Ordering::SeqCst);
17821            async move { Ok(Value::Null) }
17822        });
17823
17824        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17825        workflow.payload_codec = "json".to_string();
17826        workflow.arguments = None;
17827        let error = worker
17828            .execute_workflow_task(workflow)
17829            .expect_err("task codec must be checked before workflow invocation");
17830        assert!(error.to_string().contains("unsupported_payload_codec"));
17831
17832        let activity = ActivityTask {
17833            task_id: "activity-invalid-codec".to_string(),
17834            activity_attempt_id: None,
17835            attempt_id: None,
17836            activity_type: "codec.activity".to_string(),
17837            payload_codec: "unknown".to_string(),
17838            arguments: None,
17839            attempt_number: 1,
17840            lease_owner: None,
17841        };
17842        let error = worker
17843            .execute_activity_task(activity)
17844            .await
17845            .expect_err("task codec must be checked before activity invocation");
17846        assert!(error.to_string().contains("unsupported_payload_codec"));
17847
17848        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17849        update.workflow_update_id = Some("update-invalid-codec".to_string());
17850        update.update_name = Some("known".to_string());
17851        update.history_events.push(history_event(
17852            "UpdateAccepted",
17853            json!({
17854                "update_id": "update-invalid-codec",
17855                "update_name": "known",
17856                "arguments": {"codec": "json", "blob": null}
17857            }),
17858        ));
17859        let error = worker
17860            .execute_workflow_task(update)
17861            .expect_err("nested update codec must be checked before handler lookup");
17862        assert!(error.to_string().contains("unsupported_payload_codec"));
17863
17864        let query: QueryTask = serde_json::from_value(json!({
17865            "query_task_id": "query-invalid-codec",
17866            "workflow_type": "codec.workflow",
17867            "query_name": "known",
17868            "payload_codec": DEFAULT_CODEC,
17869            "workflow_arguments": null,
17870            "query_arguments": null,
17871            "history_export": {
17872                "payloads": {"codec": DEFAULT_CODEC},
17873                "signals": [{
17874                    "name": "empty",
17875                    "payload_codec": "json",
17876                    "arguments": null
17877                }]
17878            }
17879        }))
17880        .expect("query task");
17881        let failure = worker
17882            .execute_query_task(query)
17883            .await
17884            .expect_err("exported signal codec must be checked before query invocation");
17885        assert_eq!(failure.reason, "query_payload_decode_failed");
17886        assert!(failure.message.contains("unsupported_payload_codec"));
17887
17888        let exported_history: QueryTask = serde_json::from_value(json!({
17889            "query_task_id": "query-invalid-history-codec",
17890            "workflow_type": "codec.workflow",
17891            "query_name": "known",
17892            "payload_codec": DEFAULT_CODEC,
17893            "history_export": {
17894                "payloads": {"codec": DEFAULT_CODEC},
17895                "history_events": [{
17896                    "type": "ActivityCompleted",
17897                    "payload": {"payload_codec": "unknown", "result": null}
17898                }]
17899            }
17900        }))
17901        .expect("query task");
17902        let failure = worker
17903            .execute_query_task(exported_history)
17904            .await
17905            .expect_err("exported history codec must be checked before query invocation");
17906        assert_eq!(failure.reason, "query_payload_decode_failed");
17907        assert!(failure.message.contains("unsupported_payload_codec"));
17908        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
17909
17910        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17911        unknown_workflow.arguments = None;
17912        unknown_workflow.history_events.push(history_event(
17913            "SignalReceived",
17914            json!({
17915                "signal_name": "empty",
17916                "payload_codec": "json",
17917                "arguments": null
17918            }),
17919        ));
17920        let error = worker
17921            .execute_workflow_task(unknown_workflow)
17922            .expect_err("history codec must precede unknown workflow outcome");
17923        assert!(error.to_string().contains("unsupported_payload_codec"));
17924
17925        let unknown_activity = ActivityTask {
17926            task_id: "activity-unknown".to_string(),
17927            activity_attempt_id: None,
17928            attempt_id: None,
17929            activity_type: "missing".to_string(),
17930            payload_codec: "json".to_string(),
17931            arguments: None,
17932            attempt_number: 1,
17933            lease_owner: None,
17934        };
17935        let error = worker
17936            .execute_activity_task(unknown_activity)
17937            .await
17938            .expect_err("codec must precede unknown activity outcome");
17939        assert!(error.to_string().contains("unsupported_payload_codec"));
17940
17941        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17942        unknown_update.payload_codec = "json".to_string();
17943        unknown_update.arguments = None;
17944        unknown_update.workflow_update_id = Some("update-unknown".to_string());
17945        unknown_update.update_name = Some("missing".to_string());
17946        let error = worker
17947            .execute_workflow_task(unknown_update)
17948            .expect_err("codec must precede fail_update shortcut");
17949        assert!(error.to_string().contains("unsupported_payload_codec"));
17950
17951        let unknown_query: QueryTask = serde_json::from_value(json!({
17952            "query_task_id": "query-unknown",
17953            "workflow_type": "missing",
17954            "query_name": "missing",
17955            "payload_codec": "json",
17956            "workflow_arguments": null,
17957            "query_arguments": null
17958        }))
17959        .expect("query task");
17960        let failure = worker
17961            .execute_query_task(unknown_query)
17962            .await
17963            .expect_err("codec must precede unknown query outcome");
17964        assert_eq!(failure.reason, "query_payload_decode_failed");
17965        assert!(failure.message.contains("unsupported_payload_codec"));
17966    }
17967
17968    #[tokio::test]
17969    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
17970        let client = Client::new("http://127.0.0.1:8080").expect("client");
17971        let worker = Worker::new(client, "rust-workers");
17972
17973        for event_type in ["SignalReceived", "SignalApplied"] {
17974            for (payload_field, codec) in [
17975                ("value", "json"),
17976                ("input", "unknown"),
17977                ("arguments", "json"),
17978            ] {
17979                let payload = json!({
17980                    "signal_name": "empty",
17981                    payload_field: {"codec": codec, "blob": null}
17982                });
17983                let workflow = workflow_task(
17984                    "missing",
17985                    vec![history_event(event_type, payload.clone())],
17986                    DEFAULT_CODEC,
17987                );
17988                let error = worker
17989                    .execute_workflow_task(workflow)
17990                    .expect_err("signal payload codec must precede unknown workflow outcome");
17991                assert!(
17992                    error.to_string().contains("unsupported_payload_codec"),
17993                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
17994                );
17995
17996                let query: QueryTask = serde_json::from_value(json!({
17997                    "query_task_id": format!("query-{event_type}-{payload_field}"),
17998                    "workflow_type": "missing",
17999                    "query_name": "missing",
18000                    "payload_codec": DEFAULT_CODEC,
18001                    "workflow_arguments": null,
18002                    "query_arguments": null,
18003                    "history_events": [{
18004                        "event_type": event_type,
18005                        "payload": payload
18006                    }]
18007                }))
18008                .expect("query task");
18009                let failure = worker
18010                    .execute_query_task(query)
18011                    .await
18012                    .expect_err("signal payload codec must precede unknown query outcome");
18013                assert_eq!(
18014                    failure.reason, "query_payload_decode_failed",
18015                    "{event_type}.{payload_field} returned an unrelated query outcome"
18016                );
18017                assert!(
18018                    failure.message.contains("unsupported_payload_codec"),
18019                    "{event_type}.{payload_field} returned an unrelated query error: {}",
18020                    failure.message
18021                );
18022            }
18023        }
18024    }
18025
18026    #[test]
18027    fn workflow_context_schedules_activity_until_completion_is_in_history() {
18028        let ctx = WorkflowContext {
18029            state: Arc::new(Mutex::new(
18030                WorkflowState::new_with_identity(
18031                    Vec::new(),
18032                    Some("wf-parent".to_string()),
18033                    Some("run-parent".to_string()),
18034                    "rust-workers".to_string(),
18035                    DEFAULT_CODEC.to_string(),
18036                    None,
18037                )
18038                .expect("workflow state"),
18039            )),
18040        };
18041
18042        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18043        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18044        assert!(matches!(
18045            call.as_mut().poll(&mut task_context),
18046            Poll::Pending
18047        ));
18048
18049        let commands = ctx.take_commands().expect("commands");
18050        assert_eq!(commands[0]["type"], "schedule_activity");
18051        assert_eq!(commands[0]["activity_type"], "hello.activity");
18052    }
18053
18054    #[test]
18055    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18056        let ctx = workflow_context(Vec::new());
18057        let options = ActivityOptions::new()
18058            .task_queue("payments")
18059            .retry_policy(
18060                ActivityRetryPolicy::new(4)
18061                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18062                    .non_retryable_error_type("ValidationError"),
18063            )
18064            .start_to_close_timeout(Duration::from_secs(120))
18065            .schedule_to_start_timeout(Duration::from_secs(10))
18066            .schedule_to_close_timeout(Duration::from_secs(300))
18067            .heartbeat_timeout(Duration::from_secs(15));
18068        let mut call = Box::pin(ctx.activity_with_options(
18069            "charge-card",
18070            options,
18071            json!([{"order_id": "o-1"}]),
18072        ));
18073        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18074
18075        assert!(matches!(
18076            call.as_mut().poll(&mut task_context),
18077            Poll::Pending
18078        ));
18079        assert!(matches!(
18080            call.as_mut().poll(&mut task_context),
18081            Poll::Pending
18082        ));
18083
18084        let commands = ctx.take_commands().expect("activity command");
18085        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18086        assert_eq!(commands[0]["queue"], "payments");
18087        assert_eq!(
18088            commands[0]["retry_policy"],
18089            json!({
18090                "max_attempts": 4,
18091                "backoff_seconds": [1, 3, 9],
18092                "non_retryable_error_types": ["ValidationError"],
18093            })
18094        );
18095        assert_eq!(commands[0]["start_to_close_timeout"], 120);
18096        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18097        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18098        assert_eq!(commands[0]["heartbeat_timeout"], 15);
18099    }
18100
18101    #[test]
18102    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18103        let ctx = workflow_context(Vec::new());
18104        let options = ActivityOptions::new().retry_policy(
18105            ActivityRetryPolicy::new(3)
18106                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18107        );
18108        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18109        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18110
18111        assert!(matches!(
18112            call.as_mut().poll(&mut task_context),
18113            Poll::Pending
18114        ));
18115        assert_eq!(
18116            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18117            json!([1, 2])
18118        );
18119    }
18120
18121    #[test]
18122    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18123        let cases = [
18124            (
18125                ActivityOptions::new().task_queue("  "),
18126                ActivityOptionsErrorKind::EmptyTaskQueue,
18127            ),
18128            (
18129                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18130                ActivityOptionsErrorKind::EmptyRetryPolicy,
18131            ),
18132            (
18133                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18134                ActivityOptionsErrorKind::InvalidMaxAttempts,
18135            ),
18136            (
18137                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18138                    max_attempts: None,
18139                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18140                    non_retryable_error_types: Vec::new(),
18141                }),
18142                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18143            ),
18144            (
18145                ActivityOptions::new().retry_policy(
18146                    ActivityRetryPolicy::new(2)
18147                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18148                ),
18149                ActivityOptionsErrorKind::TooManyBackoffIntervals,
18150            ),
18151            (
18152                ActivityOptions::new().retry_policy(
18153                    ActivityRetryPolicy::new(2).exponential_backoff(
18154                        Duration::from_secs(1),
18155                        0,
18156                        None,
18157                    ),
18158                ),
18159                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18160            ),
18161            (
18162                ActivityOptions::new()
18163                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
18164                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18165            ),
18166            (
18167                ActivityOptions::new().retry_policy(
18168                    ActivityRetryPolicy::new(10_002).exponential_backoff(
18169                        Duration::from_secs(1),
18170                        1,
18171                        None,
18172                    ),
18173                ),
18174                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18175            ),
18176            (
18177                ActivityOptions::new().retry_policy(
18178                    ActivityRetryPolicy::new(2)
18179                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18180                ),
18181                ActivityOptionsErrorKind::BackoffOverflow,
18182            ),
18183        ];
18184
18185        for (options, expected_kind) in cases {
18186            let ctx = workflow_context(Vec::new());
18187            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18188            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18189            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18190                call.as_mut().poll(&mut task_context)
18191            else {
18192                panic!("expected typed activity validation error");
18193            };
18194            assert_eq!(error.kind, expected_kind);
18195            assert!(ctx.take_commands().expect("commands").is_empty());
18196        }
18197    }
18198
18199    #[test]
18200    fn activity_options_validate_positive_and_ordered_timeouts() {
18201        let zero_timeout_cases = [
18202            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18203            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18204            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18205            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18206        ];
18207        for options in zero_timeout_cases {
18208            assert_eq!(
18209                options.validate().expect_err("zero timeout").kind,
18210                ActivityOptionsErrorKind::TimeoutNotPositive
18211            );
18212        }
18213
18214        let ordering_cases = [
18215            ActivityOptions::new()
18216                .heartbeat_timeout(Duration::from_secs(11))
18217                .start_to_close_timeout(Duration::from_secs(10)),
18218            ActivityOptions::new()
18219                .start_to_close_timeout(Duration::from_secs(31))
18220                .schedule_to_close_timeout(Duration::from_secs(30)),
18221            ActivityOptions::new()
18222                .schedule_to_start_timeout(Duration::from_secs(31))
18223                .schedule_to_close_timeout(Duration::from_secs(30)),
18224        ];
18225        for options in ordering_cases {
18226            assert_eq!(
18227                options.validate().expect_err("timeout order").kind,
18228                ActivityOptionsErrorKind::TimeoutOrder
18229            );
18230        }
18231
18232        assert_eq!(
18233            ActivityOptions::new()
18234                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18235                .validate()
18236                .expect_err("protocol integer overflow")
18237                .kind,
18238            ActivityOptionsErrorKind::TimeoutOverflow
18239        );
18240    }
18241
18242    #[test]
18243    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18244        let ctx = workflow_context(completed_retry_activity_history());
18245        let mut call =
18246            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18247        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18248
18249        assert!(matches!(
18250            call.as_mut().poll(&mut task_context),
18251            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18252        ));
18253        assert!(ctx.take_commands().expect("commands").is_empty());
18254        ctx.ensure_history_consumed().expect("history consumed");
18255    }
18256
18257    #[test]
18258    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18259        let mut options = retry_activity_options();
18260        options
18261            .retry_policy
18262            .as_mut()
18263            .expect("retry policy")
18264            .non_retryable_error_types
18265            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18266
18267        let new_ctx = workflow_context(Vec::new());
18268        let mut new_call =
18269            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18270        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18271        assert!(matches!(
18272            new_call.as_mut().poll(&mut task_context),
18273            Poll::Pending
18274        ));
18275        let commands = new_ctx.take_commands().expect("commands");
18276        assert_eq!(commands.len(), 1);
18277        assert_eq!(
18278            commands[0]["retry_policy"]["non_retryable_error_types"],
18279            json!(["PermanentError"])
18280        );
18281
18282        let replay_ctx = workflow_context(completed_retry_activity_history());
18283        let mut replay_call =
18284            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18285        assert!(matches!(
18286            replay_call.as_mut().poll(&mut task_context),
18287            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18288        ));
18289        assert!(replay_ctx.take_commands().expect("commands").is_empty());
18290        replay_ctx
18291            .ensure_history_consumed()
18292            .expect("history consumed");
18293    }
18294
18295    #[test]
18296    fn replayed_intermediate_retry_remains_pending_across_restarts() {
18297        let history = completed_retry_activity_history()
18298            .into_iter()
18299            .take(3)
18300            .collect::<Vec<_>>();
18301
18302        for _restart in 0..2 {
18303            let ctx = workflow_context(history.clone());
18304            let mut call =
18305                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18306            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18307            assert!(matches!(
18308                call.as_mut().poll(&mut task_context),
18309                Poll::Pending
18310            ));
18311            assert!(ctx.take_commands().expect("commands").is_empty());
18312        }
18313    }
18314
18315    #[test]
18316    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18317        let mut changed_queue = retry_activity_options();
18318        changed_queue.task_queue = Some("different-queue".to_string());
18319
18320        let mut changed_max_attempts = retry_activity_options();
18321        let retry_policy = changed_max_attempts
18322            .retry_policy
18323            .as_mut()
18324            .expect("retry policy");
18325        retry_policy.max_attempts = Some(4);
18326
18327        let mut changed_backoff = retry_activity_options();
18328        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18329        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18330            Duration::from_secs(3),
18331            Duration::from_secs(4),
18332        ]));
18333
18334        let mut changed_non_retryable_types = retry_activity_options();
18335        let retry_policy = changed_non_retryable_types
18336            .retry_policy
18337            .as_mut()
18338            .expect("retry policy");
18339        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18340
18341        let mut changed_start_to_close = retry_activity_options();
18342        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18343        let mut changed_schedule_to_start = retry_activity_options();
18344        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18345        let mut changed_schedule_to_close = retry_activity_options();
18346        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18347        let mut changed_heartbeat = retry_activity_options();
18348        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18349
18350        let cases = [
18351            (changed_queue, "activity_task_queue_mismatch"),
18352            (changed_max_attempts, "activity_retry_policy_mismatch"),
18353            (changed_backoff, "activity_retry_policy_mismatch"),
18354            (
18355                changed_non_retryable_types,
18356                "activity_retry_policy_mismatch",
18357            ),
18358            (changed_start_to_close, "activity_retry_policy_mismatch"),
18359            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18360            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18361            (changed_heartbeat, "activity_retry_policy_mismatch"),
18362        ];
18363
18364        for (options, expected_reason) in cases {
18365            let ctx = workflow_context(completed_retry_activity_history());
18366            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18367            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18368            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18369                call.as_mut().poll(&mut task_context)
18370            else {
18371                panic!("changed activity options must fail replay");
18372            };
18373            assert_eq!(failure.reason, expected_reason);
18374            assert_eq!(failure.sequence, Some(1));
18375            assert!(ctx.take_commands().expect("commands").is_empty());
18376        }
18377    }
18378
18379    #[test]
18380    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18381        let cases = [
18382            (
18383                "execution_mode",
18384                json!("local"),
18385                "activity_execution_mode_mismatch",
18386            ),
18387            (
18388                "snapshot_version",
18389                json!(2),
18390                "activity_retry_policy_mismatch",
18391            ),
18392        ];
18393
18394        for (field, value, expected_reason) in cases {
18395            let mut history = completed_retry_activity_history();
18396            let activity = history[0].payload["activity"]
18397                .as_object_mut()
18398                .expect("activity snapshot");
18399            if field == "execution_mode" {
18400                activity.insert(field.to_string(), value);
18401            } else {
18402                activity["retry_policy"]
18403                    .as_object_mut()
18404                    .expect("retry snapshot")
18405                    .insert(field.to_string(), value);
18406            }
18407
18408            let ctx = workflow_context(history);
18409            let mut call =
18410                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18411            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18412            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18413                call.as_mut().poll(&mut task_context)
18414            else {
18415                panic!("changed {field} must fail replay");
18416            };
18417            assert_eq!(failure.reason, expected_reason);
18418            assert_eq!(failure.sequence, Some(1));
18419            assert!(ctx.take_commands().expect("commands").is_empty());
18420        }
18421    }
18422
18423    #[test]
18424    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18425        let mut history = completed_retry_activity_history();
18426        let activity = history[0].payload["activity"]
18427            .as_object_mut()
18428            .expect("activity snapshot");
18429        activity.remove("execution_mode");
18430        activity.remove("retry_policy");
18431
18432        let mut current = retry_activity_options();
18433        current.start_to_close_timeout = Some(Duration::from_secs(45));
18434        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18435        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18436        current.heartbeat_timeout = Some(Duration::from_secs(12));
18437
18438        let ctx = workflow_context(history);
18439        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18440        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18441        assert!(matches!(
18442            call.as_mut().poll(&mut task_context),
18443            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18444        ));
18445        assert!(ctx.take_commands().expect("commands").is_empty());
18446        ctx.ensure_history_consumed().expect("history consumed");
18447    }
18448
18449    #[test]
18450    fn terminal_activity_failed_after_start_returns_typed_failure() {
18451        let history = vec![
18452            history_event(
18453                "ActivityScheduled",
18454                json!({
18455                    "sequence": 1,
18456                    "activity_type": "flaky",
18457                    "activity_execution_id": "act-terminal",
18458                    "activity": {
18459                        "id": "act-terminal",
18460                        "sequence": 1,
18461                        "type": "flaky",
18462                        "queue": "critical-activities",
18463                        "retry_policy": {
18464                            "snapshot_version": 1,
18465                            "max_attempts": 3,
18466                            "backoff_seconds": [2, 4],
18467                            "non_retryable_error_types": ["PermanentError"]
18468                        }
18469                    }
18470                }),
18471            ),
18472            history_event(
18473                "ActivityStarted",
18474                json!({
18475                    "sequence": 1,
18476                    "activity_type": "flaky",
18477                    "activity_execution_id": "act-terminal",
18478                    "activity_attempt_id": "attempt-1",
18479                    "attempt_number": 1
18480                }),
18481            ),
18482            history_event(
18483                "ActivityFailed",
18484                json!({
18485                    "sequence": 1,
18486                    "activity_type": "flaky",
18487                    "activity_execution_id": "act-terminal",
18488                    "activity_attempt_id": "attempt-1",
18489                    "attempt_number": 1,
18490                    "failure_id": "failure-terminal",
18491                    "failure_category": "activity",
18492                    "exception_type": "PermanentError",
18493                    "message": "cannot retry",
18494                    "non_retryable": true
18495                }),
18496            ),
18497        ];
18498        let ctx = workflow_context(history);
18499        let mut call =
18500            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18501        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18502
18503        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18504            call.as_mut().poll(&mut task_context)
18505        else {
18506            panic!("terminal ActivityFailed must settle the activity future");
18507        };
18508        assert_eq!(failure.kind, ActivityFailureKind::Failed);
18509        assert_eq!(
18510            failure.activity_execution_id.as_deref(),
18511            Some("act-terminal")
18512        );
18513        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18514        assert!(failure.non_retryable);
18515        assert!(ctx.take_commands().expect("commands").is_empty());
18516        ctx.ensure_history_consumed().expect("history consumed");
18517    }
18518
18519    #[test]
18520    fn activity_terminal_events_return_machine_readable_failures() {
18521        let cases = [
18522            (
18523                "ActivityFailed",
18524                json!({
18525                    "sequence": 1,
18526                    "activity_type": "charge-card",
18527                    "activity_execution_id": "act-1",
18528                    "activity_attempt_id": "attempt-2",
18529                    "attempt_number": 2,
18530                    "failure_id": "failure-1",
18531                    "failure_category": "activity",
18532                    "exception_type": "PaymentDeclined",
18533                    "exception_class": "payments.PaymentDeclined",
18534                    "message": "card declined",
18535                    "non_retryable": true
18536                }),
18537                ActivityFailureKind::Failed,
18538                "activity",
18539            ),
18540            (
18541                "ActivityCancelled",
18542                json!({
18543                    "sequence": 1,
18544                    "activity_type": "charge-card",
18545                    "activity_execution_id": "act-1",
18546                    "activity_attempt_id": "attempt-1"
18547                }),
18548                ActivityFailureKind::Cancelled,
18549                "cancelled",
18550            ),
18551        ];
18552
18553        for (event_type, payload, expected_kind, expected_reason) in cases {
18554            let ctx = workflow_context(vec![history_event(event_type, payload)]);
18555            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18556            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18557            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18558                call.as_mut().poll(&mut task_context)
18559            else {
18560                panic!("expected terminal activity failure");
18561            };
18562            assert_eq!(failure.kind, expected_kind);
18563            assert_eq!(failure.reason, expected_reason);
18564            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18565            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18566        }
18567    }
18568
18569    #[test]
18570    fn every_activity_timeout_class_is_typed() {
18571        for timeout_kind in [
18572            "start_to_close",
18573            "schedule_to_start",
18574            "schedule_to_close",
18575            "heartbeat",
18576        ] {
18577            let ctx = workflow_context(vec![history_event(
18578                "ActivityTimedOut",
18579                json!({
18580                    "sequence": 1,
18581                    "activity_type": "slow",
18582                    "activity_execution_id": "act-timeout",
18583                    "activity_attempt_id": "attempt-timeout",
18584                    "failure_category": "timeout",
18585                    "timeout_kind": timeout_kind,
18586                    "message": "deadline expired"
18587                }),
18588            )]);
18589            let mut call = Box::pin(ctx.activity("slow", json!([])));
18590            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18591            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18592                call.as_mut().poll(&mut task_context)
18593            else {
18594                panic!("expected timeout failure");
18595            };
18596            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18597            assert_eq!(failure.reason, timeout_kind);
18598            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18599            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18600        }
18601    }
18602
18603    #[test]
18604    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18605        let ctx = workflow_context(Vec::new());
18606        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18607        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18608
18609        assert!(matches!(
18610            sleep.as_mut().poll(&mut task_context),
18611            Poll::Pending
18612        ));
18613        assert!(matches!(
18614            sleep.as_mut().poll(&mut task_context),
18615            Poll::Pending
18616        ));
18617
18618        let commands = ctx.take_commands().expect("timer command");
18619        assert_eq!(
18620            commands,
18621            vec![json!({
18622                "type": "start_timer",
18623                "delay_seconds": 2,
18624            })]
18625        );
18626    }
18627
18628    #[test]
18629    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18630        let history = vec![
18631            history_event(
18632                "TimerScheduled",
18633                json!({
18634                    "sequence": 1,
18635                    "timer_id": "timer-1",
18636                    "delay_seconds": 5,
18637                    "fire_at": "2026-07-11T12:00:05Z",
18638                }),
18639            ),
18640            history_event(
18641                "TimerFired",
18642                json!({
18643                    "sequence": 1,
18644                    "timer_id": "timer-1",
18645                    "delay_seconds": 5,
18646                    "fire_at": "2026-07-11T12:00:05Z",
18647                    "fired_at": "2026-07-11T12:00:05Z",
18648                }),
18649            ),
18650        ];
18651
18652        for _restart in 0..2 {
18653            let ctx = workflow_context(history.clone());
18654            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18655            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18656            assert!(matches!(
18657                sleep.as_mut().poll(&mut task_context),
18658                Poll::Ready(Ok(()))
18659            ));
18660            assert!(ctx.take_commands().expect("commands").is_empty());
18661            ctx.ensure_history_consumed().expect("history consumed");
18662        }
18663    }
18664
18665    #[test]
18666    fn workflow_sleep_rejects_changed_delay_during_replay() {
18667        let ctx = workflow_context(vec![
18668            history_event(
18669                "TimerScheduled",
18670                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18671            ),
18672            history_event(
18673                "TimerFired",
18674                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18675            ),
18676        ]);
18677        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18678        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18679
18680        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18681            sleep.as_mut().poll(&mut task_context)
18682        else {
18683            panic!("changed timer delay must be rejected");
18684        };
18685        assert_eq!(failure.reason, "timer_delay_mismatch");
18686        assert_eq!(failure.sequence, Some(1));
18687    }
18688
18689    #[test]
18690    fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18691        let ctx = workflow_context(Vec::new());
18692        let mut wait = Box::pin(
18693            ctx.wait_condition(
18694                ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18695                    .timeout(Duration::from_millis(60_001)),
18696                || Ok(false),
18697            ),
18698        );
18699        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18700
18701        assert!(matches!(
18702            wait.as_mut().poll(&mut task_context),
18703            Poll::Pending
18704        ));
18705        assert!(matches!(
18706            wait.as_mut().poll(&mut task_context),
18707            Poll::Pending
18708        ));
18709        assert_eq!(
18710            ctx.take_commands().expect("condition command"),
18711            vec![json!({
18712                "type": "open_condition_wait",
18713                "condition_wait_occurrence_id": "rust:condition-wait:0",
18714                "condition_key": "approval.ready",
18715                "condition_definition_fingerprint": "sha256:approval-v1",
18716                "timeout_seconds": 61,
18717            })]
18718        );
18719    }
18720
18721    #[test]
18722    fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18723        let ctx = workflow_context(Vec::new());
18724        let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18725        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18726        assert!(matches!(
18727            satisfied.as_mut().poll(&mut task_context),
18728            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18729        ));
18730
18731        let mut timed_out = Box::pin(wait_condition!(
18732            ctx,
18733            "no-wait",
18734            timeout: Duration::ZERO,
18735            || Ok(false),
18736        ));
18737        assert!(matches!(
18738            timed_out.as_mut().poll(&mut task_context),
18739            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18740        ));
18741        assert!(ctx.take_commands().expect("commands").is_empty());
18742    }
18743
18744    #[test]
18745    fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18746        let signal_history = vec![
18747            history_event(
18748                "ConditionWaitOpened",
18749                json!({
18750                    "sequence": 4,
18751                    "condition_wait_id": "condition:4",
18752                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18753                    "condition_key": "approval",
18754                    "condition_definition_fingerprint": "sha256:approval-v1",
18755                    "timeout_seconds": 30,
18756                }),
18757            ),
18758            history_event(
18759                "SignalReceived",
18760                json!({
18761                    "workflow_sequence": 4,
18762                    "signal_name": "approve",
18763                    "arguments": fixture_envelope(json!(["Ada"])),
18764                }),
18765            ),
18766        ];
18767        for _worker_before_or_after_restart in 0..2 {
18768            let ctx = workflow_context(signal_history.clone());
18769            let predicate_ctx = ctx.clone();
18770            let mut wait = Box::pin(
18771                ctx.wait_condition(
18772                    ConditionWaitOptions::new("approval", "sha256:approval-v1")
18773                        .timeout(Duration::from_secs(30)),
18774                    move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18775                ),
18776            );
18777            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18778            assert!(matches!(
18779                wait.as_mut().poll(&mut task_context),
18780                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18781            ));
18782            assert!(ctx.take_commands().expect("commands").is_empty());
18783            ctx.ensure_history_consumed().expect("condition consumed");
18784        }
18785
18786        let update_history = vec![
18787            history_event(
18788                "ConditionWaitOpened",
18789                json!({
18790                    "sequence": 7,
18791                    "condition_wait_id": "condition:7",
18792                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18793                    "condition_key": "update-approval",
18794                    "condition_definition_fingerprint": "sha256:update-approval-v1",
18795                }),
18796            ),
18797            history_event(
18798                "UpdateApplied",
18799                json!({
18800                    "sequence": 7,
18801                    "update_id": "update-1",
18802                    "update_name": "approve",
18803                    "arguments": fixture_envelope(json!([true])),
18804                }),
18805            ),
18806        ];
18807        let ctx = workflow_context(update_history);
18808        let predicate_ctx = ctx.clone();
18809        let mut wait = Box::pin(ctx.wait_condition(
18810            ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18811            move || {
18812                Ok(predicate_ctx
18813                    .updates("approve")?
18814                    .first()
18815                    .and_then(|arguments| arguments.first())
18816                    .and_then(Value::as_bool)
18817                    == Some(true))
18818            },
18819        ));
18820        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18821        assert!(matches!(
18822            wait.as_mut().poll(&mut task_context),
18823            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18824        ));
18825        assert!(ctx.take_commands().expect("commands").is_empty());
18826        ctx.ensure_history_consumed().expect("condition consumed");
18827    }
18828
18829    #[test]
18830    fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18831        let open_history = vec![
18832            history_event(
18833                "ConditionWaitOpened",
18834                json!({
18835                    "sequence": 3,
18836                    "condition_wait_id": "condition:3",
18837                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18838                    "condition_key": "two-votes",
18839                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18840                    "timeout_seconds": 120,
18841                }),
18842            ),
18843            history_event(
18844                "SignalReceived",
18845                json!({
18846                    "workflow_sequence": 3,
18847                    "signal_name": "vote",
18848                    "arguments": fixture_envelope(json!(["first"])),
18849                }),
18850            ),
18851        ];
18852        for _worker_before_or_after_restart in 0..2 {
18853            let ctx = workflow_context(open_history.clone());
18854            let predicate_ctx = ctx.clone();
18855            let mut wait = Box::pin(
18856                ctx.wait_condition(
18857                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18858                        .timeout(Duration::from_secs(120)),
18859                    move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18860                ),
18861            );
18862            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18863            assert!(matches!(
18864                wait.as_mut().poll(&mut task_context),
18865                Poll::Pending
18866            ));
18867            assert_eq!(
18868                ctx.take_commands().expect("reopened condition"),
18869                vec![json!({
18870                    "type": "open_condition_wait",
18871                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18872                    "condition_key": "two-votes",
18873                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18874                    "timeout_seconds": 120,
18875                })]
18876            );
18877        }
18878
18879        let satisfied_ctx = workflow_context(vec![
18880            history_event(
18881                "ConditionWaitOpened",
18882                json!({
18883                    "sequence": 5,
18884                    "condition_wait_id": "condition:5",
18885                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18886                    "condition_key": "approval",
18887                    "condition_definition_fingerprint": "sha256:approval-v1",
18888                }),
18889            ),
18890            history_event(
18891                "ConditionWaitSatisfied",
18892                json!({
18893                    "sequence": 5,
18894                    "condition_wait_id": "condition:5",
18895                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18896                    "condition_key": "approval",
18897                    "condition_definition_fingerprint": "sha256:approval-v1",
18898                }),
18899            ),
18900        ]);
18901        let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18902            ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18903            || Ok(false),
18904        ));
18905        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18906        assert!(matches!(
18907            satisfied.as_mut().poll(&mut task_context),
18908            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18909        ));
18910
18911        let timed_out_ctx = workflow_context(vec![
18912            history_event(
18913                "ConditionWaitOpened",
18914                json!({
18915                    "sequence": 8,
18916                    "condition_wait_id": "condition:8",
18917                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18918                    "condition_key": "approval-timeout",
18919                    "condition_definition_fingerprint": "sha256:approval-timeout-v1",
18920                    "timeout_seconds": 5,
18921                }),
18922            ),
18923            history_event(
18924                "TimerScheduled",
18925                json!({
18926                    "sequence": 9,
18927                    "timer_id": "condition-timer:9",
18928                    "timer_kind": "condition_timeout",
18929                    "condition_wait_id": "condition:8",
18930                    "delay_seconds": 5,
18931                }),
18932            ),
18933            history_event(
18934                "TimerFired",
18935                json!({
18936                    "sequence": 9,
18937                    "timer_id": "condition-timer:9",
18938                    "timer_kind": "condition_timeout",
18939                    "condition_wait_id": "condition:8",
18940                    "delay_seconds": 5,
18941                }),
18942            ),
18943        ]);
18944        let mut timed_out = Box::pin(
18945            timed_out_ctx.wait_condition(
18946                ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
18947                    .timeout(Duration::from_secs(5)),
18948                || Ok(true),
18949            ),
18950        );
18951        assert!(matches!(
18952            timed_out.as_mut().poll(&mut task_context),
18953            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18954        ));
18955    }
18956
18957    #[test]
18958    fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
18959        let history = vec![
18960            history_event(
18961                "ConditionWaitOpened",
18962                json!({
18963                    "sequence": 3,
18964                    "condition_wait_id": "condition:3",
18965                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18966                    "condition_key": "two-votes",
18967                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18968                }),
18969            ),
18970            history_event(
18971                "SignalReceived",
18972                json!({
18973                    "workflow_sequence": 3,
18974                    "signal_name": "vote",
18975                    "arguments": fixture_envelope(json!(["first"])),
18976                }),
18977            ),
18978            history_event(
18979                "ConditionWaitSatisfied",
18980                json!({
18981                    "sequence": 3,
18982                    "condition_wait_id": "condition:3",
18983                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18984                    "condition_key": "two-votes",
18985                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18986                }),
18987            ),
18988            history_event(
18989                "ConditionWaitOpened",
18990                json!({
18991                    "sequence": 5,
18992                    "condition_wait_id": "condition:5",
18993                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18994                    "condition_key": "two-votes",
18995                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18996                }),
18997            ),
18998            history_event(
18999                "SignalReceived",
19000                json!({
19001                    "workflow_sequence": 5,
19002                    "signal_name": "vote",
19003                    "arguments": fixture_envelope(json!(["second"])),
19004                }),
19005            ),
19006            history_event(
19007                "ConditionWaitSatisfied",
19008                json!({
19009                    "sequence": 5,
19010                    "condition_wait_id": "condition:5",
19011                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19012                    "condition_key": "two-votes",
19013                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19014                }),
19015            ),
19016        ];
19017        for _cold_worker_or_restart in 0..2 {
19018            let ctx = workflow_context(history.clone());
19019            let predicate_ctx = ctx.clone();
19020            let mut wait = Box::pin(ctx.wait_condition(
19021                ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19022                move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19023            ));
19024            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19025
19026            assert!(matches!(
19027                wait.as_mut().poll(&mut task_context),
19028                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19029            ));
19030            assert!(ctx.take_commands().expect("commands").is_empty());
19031            ctx.ensure_history_consumed()
19032                .expect("every physical wait-open is consumed");
19033        }
19034    }
19035
19036    #[test]
19037    fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19038        let history = vec![
19039            history_event(
19040                "ConditionWaitOpened",
19041                json!({
19042                    "sequence": 3,
19043                    "condition_wait_id": "condition:3",
19044                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19045                    "condition_key": "approved",
19046                    "condition_definition_fingerprint": "sha256:approved-v1",
19047                }),
19048            ),
19049            history_event(
19050                "UpdateApplied",
19051                json!({
19052                    "sequence": 3,
19053                    "update_id": "update-1",
19054                    "update_name": "approve",
19055                    "arguments": fixture_envelope(json!([false])),
19056                }),
19057            ),
19058            history_event(
19059                "ConditionWaitOpened",
19060                json!({
19061                    "sequence": 5,
19062                    "condition_wait_id": "condition:5",
19063                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19064                    "condition_key": "approved",
19065                    "condition_definition_fingerprint": "sha256:approved-v1",
19066                }),
19067            ),
19068            history_event(
19069                "UpdateApplied",
19070                json!({
19071                    "sequence": 5,
19072                    "update_id": "update-2",
19073                    "update_name": "approve",
19074                    "arguments": fixture_envelope(json!([true])),
19075                }),
19076            ),
19077        ];
19078
19079        for _cold_worker_or_restart in 0..2 {
19080            let ctx = workflow_context(history.clone());
19081            let predicate_ctx = ctx.clone();
19082            let mut wait = Box::pin(ctx.wait_condition(
19083                ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19084                move || {
19085                    Ok(predicate_ctx
19086                        .updates("approve")?
19087                        .last()
19088                        .and_then(|arguments| arguments.first())
19089                        .and_then(Value::as_bool)
19090                        == Some(true))
19091                },
19092            ));
19093            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19094
19095            assert!(matches!(
19096                wait.as_mut().poll(&mut task_context),
19097                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19098            ));
19099            assert!(ctx.take_commands().expect("commands").is_empty());
19100            ctx.ensure_history_consumed()
19101                .expect("every update-driven reopen is consumed");
19102        }
19103    }
19104
19105    #[test]
19106    fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19107        for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19108            ("shared", "sha256:first", "shared", "sha256:second"),
19109            ("first", "sha256:shared", "second", "sha256:shared"),
19110            ("shared", "sha256:shared", "shared", "sha256:shared"),
19111            ("first", "sha256:first", "second", "sha256:second"),
19112        ] {
19113            let history = vec![
19114                history_event(
19115                    "ConditionWaitOpened",
19116                    json!({
19117                        "sequence": 3,
19118                        "condition_wait_id": "condition:3",
19119                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19120                        "condition_key": first_key,
19121                        "condition_definition_fingerprint": first_fingerprint,
19122                    }),
19123                ),
19124                history_event(
19125                    "ConditionWaitSatisfied",
19126                    json!({
19127                        "sequence": 3,
19128                        "condition_wait_id": "condition:3",
19129                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19130                        "condition_key": first_key,
19131                        "condition_definition_fingerprint": first_fingerprint,
19132                    }),
19133                ),
19134                history_event(
19135                    "ConditionWaitOpened",
19136                    json!({
19137                        "sequence": 4,
19138                        "condition_wait_id": "condition:4",
19139                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19140                        "condition_key": second_key,
19141                        "condition_definition_fingerprint": second_fingerprint,
19142                    }),
19143                ),
19144                history_event(
19145                    "ConditionWaitSatisfied",
19146                    json!({
19147                        "sequence": 4,
19148                        "condition_wait_id": "condition:4",
19149                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19150                        "condition_key": second_key,
19151                        "condition_definition_fingerprint": second_fingerprint,
19152                    }),
19153                ),
19154            ];
19155            for _cold_worker_or_restart in 0..2 {
19156                let ctx = workflow_context(history.clone());
19157                let mut task_context = TaskContext::from_waker(noop_waker_ref());
19158                let mut first = Box::pin(ctx.wait_condition(
19159                    ConditionWaitOptions::new(first_key, first_fingerprint),
19160                    || Ok(false),
19161                ));
19162                assert!(matches!(
19163                    first.as_mut().poll(&mut task_context),
19164                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19165                ));
19166
19167                let mut second = Box::pin(ctx.wait_condition(
19168                    ConditionWaitOptions::new(second_key, second_fingerprint),
19169                    || Ok(false),
19170                ));
19171                assert!(matches!(
19172                    second.as_mut().poll(&mut task_context),
19173                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19174                ));
19175                assert!(ctx.take_commands().expect("commands").is_empty());
19176                ctx.ensure_history_consumed()
19177                    .expect("each authored wait consumes one occurrence");
19178            }
19179        }
19180    }
19181
19182    #[test]
19183    fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19184        fn worker() -> Worker {
19185            let client = Client::new("http://127.0.0.1:8080").expect("client");
19186            let mut worker = Worker::new(client, "rust-workers");
19187            worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19188                let mut outcomes = Vec::new();
19189                for _ in 0..2 {
19190                    outcomes.push(
19191                        ctx.wait_condition(
19192                            ConditionWaitOptions::new("shared", "sha256:shared"),
19193                            || Ok(false),
19194                        )
19195                        .await?,
19196                    );
19197                }
19198                Ok(json!(outcomes))
19199            });
19200            worker
19201        }
19202
19203        let task = workflow_task(
19204            "rust.condition-loop",
19205            vec![
19206                history_event(
19207                    "ConditionWaitOpened",
19208                    json!({
19209                        "sequence": 1,
19210                        "condition_wait_id": "condition:1",
19211                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19212                        "condition_key": "shared",
19213                        "condition_definition_fingerprint": "sha256:shared",
19214                    }),
19215                ),
19216                history_event(
19217                    "ConditionWaitSatisfied",
19218                    json!({
19219                        "sequence": 1,
19220                        "condition_wait_id": "condition:1",
19221                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19222                        "condition_key": "shared",
19223                        "condition_definition_fingerprint": "sha256:shared",
19224                    }),
19225                ),
19226                history_event(
19227                    "ConditionWaitOpened",
19228                    json!({
19229                        "sequence": 2,
19230                        "condition_wait_id": "condition:2",
19231                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19232                        "condition_key": "shared",
19233                        "condition_definition_fingerprint": "sha256:shared",
19234                    }),
19235                ),
19236                history_event(
19237                    "ConditionWaitSatisfied",
19238                    json!({
19239                        "sequence": 2,
19240                        "condition_wait_id": "condition:2",
19241                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19242                        "condition_key": "shared",
19243                        "condition_definition_fingerprint": "sha256:shared",
19244                    }),
19245                ),
19246            ],
19247            DEFAULT_CODEC,
19248        );
19249
19250        for _cold_worker_or_restart in 0..2 {
19251            let commands = worker()
19252                .execute_workflow_task(task.clone())
19253                .expect("adjacent loop waits replay deterministically");
19254            assert_eq!(commands.len(), 1);
19255            assert_eq!(commands[0]["type"], "complete_workflow");
19256            assert_eq!(
19257                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19258                json!(["satisfied", "satisfied"])
19259            );
19260        }
19261    }
19262
19263    #[test]
19264    fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19265        let history = vec![history_event(
19266            "ConditionWaitOpened",
19267            json!({
19268                "sequence": 12,
19269                "condition_wait_id": "condition:12",
19270                "condition_wait_occurrence_id": "rust:condition-wait:0",
19271                "condition_key": "approval",
19272                "condition_definition_fingerprint": "sha256:approval-v1",
19273                "timeout_seconds": 30,
19274            }),
19275        )];
19276        for (options, expected_reason) in [
19277            (
19278                ConditionWaitOptions::new("changed", "sha256:approval-v1")
19279                    .timeout(Duration::from_secs(30)),
19280                "condition_wait_key_mismatch",
19281            ),
19282            (
19283                ConditionWaitOptions::new("approval", "sha256:approval-v2")
19284                    .timeout(Duration::from_secs(30)),
19285                "condition_wait_predicate_mismatch",
19286            ),
19287            (
19288                ConditionWaitOptions::new("approval", "sha256:approval-v1")
19289                    .timeout(Duration::from_secs(29)),
19290                "condition_wait_timeout_mismatch",
19291            ),
19292        ] {
19293            let ctx = workflow_context(history.clone());
19294            let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19295            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19296            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19297                wait.as_mut().poll(&mut task_context)
19298            else {
19299                panic!("changed condition definition must fail replay");
19300            };
19301            assert_eq!(failure.reason, expected_reason);
19302            assert_eq!(failure.sequence, Some(12));
19303        }
19304    }
19305
19306    #[test]
19307    fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19308        let error = WorkflowState::new(
19309            vec![history_event(
19310                "ConditionWaitOpened",
19311                json!({
19312                    "sequence": 12,
19313                    "condition_wait_id": "condition:12",
19314                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19315                    "condition_key": "approval",
19316                }),
19317            )],
19318            "rust-workers".to_string(),
19319            DEFAULT_CODEC.to_string(),
19320            None,
19321        )
19322        .expect_err("condition history without a predicate fingerprint must fail");
19323
19324        assert!(matches!(
19325            error,
19326            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19327                if reason == "condition_wait_predicate_fingerprint_missing"
19328        ));
19329    }
19330
19331    #[test]
19332    fn condition_wait_history_requires_authored_occurrence_identity() {
19333        let error = WorkflowState::new(
19334            vec![history_event(
19335                "ConditionWaitOpened",
19336                json!({
19337                    "sequence": 12,
19338                    "condition_wait_id": "condition:12",
19339                    "condition_key": "approval",
19340                    "condition_definition_fingerprint": "sha256:approval-v1",
19341                }),
19342            )],
19343            "rust-workers".to_string(),
19344            DEFAULT_CODEC.to_string(),
19345            None,
19346        )
19347        .expect_err("condition history without occurrence identity must fail");
19348
19349        assert!(matches!(
19350            error,
19351            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19352                if reason == "condition_wait_occurrence_id_missing"
19353        ));
19354    }
19355
19356    #[test]
19357    fn typed_search_attribute_updates_validate_emit_and_replay() {
19358        let update = SearchAttributeUpdate::new()
19359            .keyword("OrderStatus", " waiting ")
19360            .expect("keyword")
19361            .int("Attempt", 3)
19362            .expect("int")
19363            .bool("Escalated", false)
19364            .expect("bool")
19365            .keyword_list("Regions", ["us-east", "eu-west"])
19366            .expect("list")
19367            .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19368            .expect("datetime")
19369            .delete("LegacyStatus")
19370            .expect("delete");
19371        let ctx = workflow_context(Vec::new());
19372        ctx.upsert_search_attributes(update.clone())
19373            .expect("typed update");
19374        assert_eq!(
19375            ctx.take_commands().expect("search-attribute command"),
19376            vec![json!({
19377                "type": "upsert_search_attributes",
19378                "attributes": {
19379                    "Attempt": 3,
19380                    "Escalated": false,
19381                    "LegacyStatus": null,
19382                    "OrderStatus": "waiting",
19383                    "Regions": ["us-east", "eu-west"],
19384                    "UpdatedAt": "2026-08-22T04:00:00Z",
19385                },
19386                "attribute_types": {
19387                    "Attempt": "int",
19388                    "Escalated": "bool",
19389                    "OrderStatus": "keyword",
19390                    "Regions": "keyword_list",
19391                    "UpdatedAt": "datetime",
19392                },
19393            })]
19394        );
19395
19396        let replay = workflow_context(vec![history_event(
19397            "SearchAttributesUpserted",
19398            json!({
19399                "sequence": 6,
19400                "attributes": {
19401                    "Attempt": 3,
19402                    "Escalated": false,
19403                    "LegacyStatus": null,
19404                    "OrderStatus": "waiting",
19405                    "Regions": ["us-east", "eu-west"],
19406                    "UpdatedAt": "2026-08-22T04:00:00Z",
19407                },
19408                "attribute_types": {
19409                    "Attempt": "int",
19410                    "Escalated": "bool",
19411                    "OrderStatus": "keyword",
19412                    "Regions": "keyword_list",
19413                    "UpdatedAt": "datetime",
19414                },
19415                "merged": {},
19416            }),
19417        )]);
19418        replay
19419            .upsert_search_attributes(update)
19420            .expect("matching update replays");
19421        assert!(replay.take_commands().expect("commands").is_empty());
19422        replay.ensure_history_consumed().expect("history consumed");
19423
19424        let type_drift = workflow_context(vec![history_event(
19425            "SearchAttributesUpserted",
19426            json!({
19427                "sequence": 7,
19428                "attributes": {"OrderStatus": "waiting"},
19429                "attribute_types": {"OrderStatus": "keyword"},
19430                "merged": {"OrderStatus": "waiting"},
19431            }),
19432        )]);
19433        let error = type_drift
19434            .upsert_search_attributes(
19435                SearchAttributeUpdate::new()
19436                    .string("OrderStatus", "waiting")
19437                    .expect("string update"),
19438            )
19439            .expect_err("same JSON value with a changed type must fail replay");
19440        assert!(matches!(
19441            error,
19442            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19443                if reason == "search_attribute_type_mismatch"
19444        ));
19445
19446        let malformed_types = WorkflowState::new(
19447            vec![history_event(
19448                "SearchAttributesUpserted",
19449                json!({
19450                    "sequence": 8,
19451                    "attributes": {"OrderStatus": "waiting"},
19452                    "attribute_types": {"OrderStatus": "unsupported"},
19453                    "merged": {"OrderStatus": "waiting"},
19454                }),
19455            )],
19456            "rust-workers".to_string(),
19457            DEFAULT_CODEC.to_string(),
19458            None,
19459        )
19460        .expect_err("unsupported search-attribute type metadata must fail");
19461        assert!(matches!(
19462            malformed_types,
19463            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19464                if reason == "search_attribute_types_malformed"
19465        ));
19466
19467        assert!(matches!(
19468            SearchAttributeUpdate::new().keyword("bad key", "value"),
19469            Err(SearchAttributeUpdateError::InvalidKey(_))
19470        ));
19471        assert!(matches!(
19472            SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19473            Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19474        ));
19475        assert!(matches!(
19476            SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19477            Err(SearchAttributeUpdateError::ValueTooLong { .. })
19478        ));
19479        assert!(matches!(
19480            SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19481            Err(SearchAttributeUpdateError::InvalidDateTime(_))
19482        ));
19483        assert!(matches!(
19484            workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19485            Err(Error::InvalidSearchAttributeUpdate(
19486                SearchAttributeUpdateError::Empty
19487            ))
19488        ));
19489    }
19490
19491    #[test]
19492    fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19493        let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19494        let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19495
19496        assert!(SearchAttributeUpdate::new()
19497            .string("AsciiDescription", ascii)
19498            .is_ok());
19499        assert!(SearchAttributeUpdate::new()
19500            .string("Utf8Description", utf8)
19501            .is_ok());
19502        assert!(matches!(
19503            SearchAttributeUpdate::new().string(
19504                "TooLongDescription",
19505                "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19506            ),
19507            Err(SearchAttributeUpdateError::ValueTooLong {
19508                kind: "string",
19509                limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19510                ..
19511            })
19512        ));
19513    }
19514
19515    #[test]
19516    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19517        let lone_fire = WorkflowState::new(
19518            vec![history_event(
19519                "TimerFired",
19520                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19521            )],
19522            "rust-workers".to_string(),
19523            DEFAULT_CODEC.to_string(),
19524            None,
19525        )
19526        .expect_err("TimerFired requires TimerScheduled");
19527        assert!(matches!(
19528            lone_fire,
19529            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19530                if reason == "timer_schedule_missing_or_duplicate"
19531        ));
19532
19533        let wrong_identity = WorkflowState::new(
19534            vec![
19535                history_event(
19536                    "TimerScheduled",
19537                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19538                ),
19539                history_event(
19540                    "TimerFired",
19541                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19542                ),
19543            ],
19544            "rust-workers".to_string(),
19545            DEFAULT_CODEC.to_string(),
19546            None,
19547        )
19548        .expect_err("fire must match scheduled timer identity");
19549        assert!(matches!(
19550            wrong_identity,
19551            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19552                if reason == "timer_identity_mismatch"
19553        ));
19554
19555        let duplicate_fire = WorkflowState::new(
19556            vec![
19557                history_event(
19558                    "TimerScheduled",
19559                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19560                ),
19561                history_event(
19562                    "TimerFired",
19563                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19564                ),
19565                history_event(
19566                    "TimerFired",
19567                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19568                ),
19569            ],
19570            "rust-workers".to_string(),
19571            DEFAULT_CODEC.to_string(),
19572            None,
19573        )
19574        .expect_err("a durable timer cannot fire twice");
19575        assert!(matches!(
19576            duplicate_fire,
19577            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19578                if reason == "duplicate_timer_fire"
19579        ));
19580
19581        let wrong_fired_delay = WorkflowState::new(
19582            vec![
19583                history_event(
19584                    "TimerScheduled",
19585                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19586                ),
19587                history_event(
19588                    "TimerFired",
19589                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19590                ),
19591            ],
19592            "rust-workers".to_string(),
19593            DEFAULT_CODEC.to_string(),
19594            None,
19595        )
19596        .expect_err("timer schedule and fire delays must agree");
19597        assert!(matches!(
19598            wrong_fired_delay,
19599            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19600                if reason == "timer_history_delay_mismatch"
19601        ));
19602    }
19603
19604    #[test]
19605    fn replay_rejects_activity_moved_before_recorded_timer() {
19606        let ctx = workflow_context(vec![
19607            history_event(
19608                "TimerScheduled",
19609                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19610            ),
19611            history_event(
19612                "TimerFired",
19613                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19614            ),
19615            history_event(
19616                "ActivityCompleted",
19617                json!({
19618                    "sequence": 2,
19619                    "activity_type": "after-timer",
19620                    "payload_codec": DEFAULT_CODEC,
19621                    "result": fixture_envelope(json!("done")),
19622                }),
19623            ),
19624        ]);
19625        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19626        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19627
19628        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19629            activity.as_mut().poll(&mut task_context)
19630        else {
19631            panic!("reordered durable command must be rejected");
19632        };
19633        assert_eq!(failure.reason, "recorded_command_mismatch");
19634        assert_eq!(failure.sequence, Some(1));
19635        assert_eq!(failure.expected.as_deref(), Some("timer"));
19636        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19637    }
19638
19639    #[test]
19640    fn workflow_context_emits_a_typed_named_signal_wait() {
19641        let ctx = workflow_context(Vec::new());
19642        let mut signal = Box::pin(ctx.wait_signal("finish"));
19643        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19644
19645        assert!(matches!(
19646            signal.as_mut().poll(&mut task_context),
19647            Poll::Pending
19648        ));
19649        assert_eq!(
19650            ctx.take_commands().expect("signal-wait command"),
19651            vec![json!({
19652                "type": "open_signal_wait",
19653                "signal_name": "finish",
19654            })]
19655        );
19656    }
19657
19658    #[test]
19659    fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19660        let ctx = workflow_context(Vec::new());
19661        let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19662        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19663
19664        let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19665        else {
19666            panic!("runtime-reserved signal should be rejected");
19667        };
19668        assert!(message.contains("reserved by the workflow runtime"));
19669        assert!(ctx.take_commands().expect("commands").is_empty());
19670    }
19671
19672    #[tokio::test]
19673    async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19674        let client = Client::builder("http://127.0.0.1:9")
19675            .build()
19676            .expect("client");
19677        let error = client
19678            .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19679            .await
19680            .expect_err("runtime-reserved signal should be rejected before transport");
19681
19682        assert!(
19683            matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19684        );
19685    }
19686
19687    #[test]
19688    fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19689        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19690            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19691                value.to_string(),
19692            )]))
19693            .expect("message payload");
19694            json!({
19695                "schema": MESSAGE_STREAM_SCHEMA,
19696                "stream_name": "orders",
19697                "message_id": message_id,
19698                "position": position,
19699                "payload_envelope": payload,
19700            })
19701        }
19702
19703        fn opened(sequence: u64) -> HistoryEvent {
19704            history_event(
19705                "SignalWaitOpened",
19706                json!({
19707                    "sequence": sequence,
19708                    "signal_name": MESSAGE_STREAM_SIGNAL,
19709                }),
19710            )
19711        }
19712
19713        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19714            history_event(
19715                "SignalApplied",
19716                json!({
19717                    "sequence": sequence,
19718                    "signal_name": MESSAGE_STREAM_SIGNAL,
19719                    "value": fixture_envelope(json!([delivery])),
19720                }),
19721            )
19722        }
19723
19724        fn received(delivery: Value) -> HistoryEvent {
19725            history_event(
19726                "SignalReceived",
19727                json!({
19728                    "signal_name": MESSAGE_STREAM_SIGNAL,
19729                    "arguments": fixture_envelope(json!([delivery])),
19730                    "payload_codec": DEFAULT_CODEC,
19731                }),
19732            )
19733        }
19734
19735        let client = Client::new("http://127.0.0.1:8080").expect("client");
19736        let mut worker = Worker::new(client, "rust-workers");
19737        worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19738            let messages = ctx.message_stream("orders")?.receive(2).await?;
19739            Ok(json!(messages
19740                .into_iter()
19741                .map(|message| message.message_id)
19742                .collect::<Vec<_>>()))
19743        });
19744
19745        let first = delivery("message-1", 1, "one");
19746        let second = delivery("message-2", 2, "two");
19747        let batch = worker
19748            .execute_workflow_task_decision(workflow_task(
19749                "rust.message-stream-batch",
19750                vec![
19751                    opened(1),
19752                    received(first.clone()),
19753                    applied(1, first.clone()),
19754                    received(first.clone()),
19755                    received(second),
19756                ],
19757                DEFAULT_CODEC,
19758            ))
19759            .expect("worker task consumes the available batch");
19760
19761        assert_eq!(batch.commands.len(), 1);
19762        assert_eq!(batch.commands[0]["type"], "complete_workflow");
19763        assert_eq!(
19764            decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19765                .expect("workflow result"),
19766            json!(["message-1", "message-2"])
19767        );
19768        assert_eq!(
19769            batch.message_stream_cursors,
19770            vec![json!({"stream_name": "orders", "through_position": 2})]
19771        );
19772        assert!(batch.message_stream_waits.is_empty());
19773
19774        let partial = worker
19775            .execute_workflow_task_decision(workflow_task(
19776                "rust.message-stream-batch",
19777                vec![opened(1), received(first.clone()), applied(1, first)],
19778                DEFAULT_CODEC,
19779            ))
19780            .expect("worker task returns without waiting for a missing second item");
19781        assert_eq!(partial.commands.len(), 1);
19782        assert_eq!(partial.commands[0]["type"], "complete_workflow");
19783        assert_eq!(
19784            decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19785                .expect("workflow result"),
19786            json!(["message-1"])
19787        );
19788        assert_eq!(
19789            partial.message_stream_cursors,
19790            vec![json!({"stream_name": "orders", "through_position": 1})]
19791        );
19792        assert!(partial.message_stream_waits.is_empty());
19793    }
19794
19795    #[test]
19796    fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19797        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19798            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19799                value.to_string(),
19800            )]))
19801            .expect("message payload");
19802            json!({
19803                "schema": MESSAGE_STREAM_SCHEMA,
19804                "stream_name": "orders",
19805                "message_id": message_id,
19806                "position": position,
19807                "payload_envelope": payload,
19808            })
19809        }
19810
19811        fn opened(sequence: u64) -> HistoryEvent {
19812            history_event(
19813                "SignalWaitOpened",
19814                json!({
19815                    "sequence": sequence,
19816                    "signal_name": MESSAGE_STREAM_SIGNAL,
19817                }),
19818            )
19819        }
19820
19821        fn received(delivery: Value) -> HistoryEvent {
19822            history_event(
19823                "SignalReceived",
19824                json!({
19825                    "signal_name": MESSAGE_STREAM_SIGNAL,
19826                    "arguments": fixture_envelope(json!([delivery])),
19827                    "payload_codec": DEFAULT_CODEC,
19828                }),
19829            )
19830        }
19831
19832        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19833            history_event(
19834                "SignalApplied",
19835                json!({
19836                    "sequence": sequence,
19837                    "signal_name": MESSAGE_STREAM_SIGNAL,
19838                    "value": fixture_envelope(json!([delivery])),
19839                }),
19840            )
19841        }
19842
19843        let client = Client::new("http://127.0.0.1:8080").expect("client");
19844        let mut worker = Worker::new(client, "rust-workers");
19845        worker.register_workflow(
19846            "rust.message-stream-partial-batches",
19847            |ctx, _input| async move {
19848                let stream = ctx.message_stream("orders")?;
19849                let first = stream.receive(10).await?;
19850                let second = stream.receive(10).await?;
19851                Ok(json!([
19852                    first
19853                        .into_iter()
19854                        .map(|message| message.message_id)
19855                        .collect::<Vec<_>>(),
19856                    second
19857                        .into_iter()
19858                        .map(|message| message.message_id)
19859                        .collect::<Vec<_>>(),
19860                ]))
19861            },
19862        );
19863
19864        let first = delivery("message-1", 1, "one");
19865        let second = delivery("message-2", 2, "two");
19866        let decision = worker
19867            .execute_workflow_task_decision(workflow_task(
19868                "rust.message-stream-partial-batches",
19869                vec![
19870                    opened(1),
19871                    received(first.clone()),
19872                    applied(1, first),
19873                    opened(2),
19874                    received(second.clone()),
19875                    applied(2, second),
19876                ],
19877                DEFAULT_CODEC,
19878            ))
19879            .expect("cold replay preserves both authored receive boundaries");
19880
19881        assert_eq!(decision.commands.len(), 1);
19882        assert_eq!(decision.commands[0]["type"], "complete_workflow");
19883        assert_eq!(
19884            decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19885                .expect("workflow result"),
19886            json!([["message-1"], ["message-2"]])
19887        );
19888        assert_eq!(
19889            decision.message_stream_cursors,
19890            vec![json!({"stream_name": "orders", "through_position": 2})]
19891        );
19892        assert!(decision.message_stream_waits.is_empty());
19893    }
19894
19895    #[test]
19896    fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19897        let ctx = workflow_context(Vec::new());
19898        let stream = ctx.message_stream("orders").expect("message stream");
19899        let mut receive = Box::pin(stream.receive(10));
19900        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19901
19902        assert!(matches!(
19903            receive.as_mut().poll(&mut task_context),
19904            Poll::Pending
19905        ));
19906        assert_eq!(
19907            ctx.take_commands().expect("message-stream wait command"),
19908            vec![json!({
19909                "type": "open_signal_wait",
19910                "signal_name": MESSAGE_STREAM_SIGNAL,
19911            })]
19912        );
19913        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19914        assert!(cursors.is_empty());
19915        assert_eq!(
19916            waits,
19917            vec![json!({"stream_name": "orders", "after_position": 0})]
19918        );
19919    }
19920
19921    #[test]
19922    fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
19923        let ctx = workflow_context(vec![history_event(
19924            "SignalReceived",
19925            json!({
19926                "signal_name": MESSAGE_STREAM_SIGNAL,
19927                "arguments": fixture_envelope(json!([{
19928                    "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
19929                    "stream_name": "orders",
19930                    "through_position": 2,
19931                }])),
19932                "payload_codec": DEFAULT_CODEC,
19933            }),
19934        )]);
19935        let stream = ctx.message_stream("orders").expect("message stream");
19936        let mut receive = Box::pin(stream.receive(10));
19937        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19938
19939        assert!(matches!(
19940            receive.as_mut().poll(&mut task_context),
19941            Poll::Pending
19942        ));
19943        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19944        assert_eq!(
19945            cursors,
19946            vec![json!({"stream_name": "orders", "through_position": 2})]
19947        );
19948        assert_eq!(
19949            waits,
19950            vec![json!({"stream_name": "orders", "after_position": 2})]
19951        );
19952    }
19953
19954    #[test]
19955    fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
19956        let mut empty_map = BTreeMap::new();
19957        let mut nested = BTreeMap::new();
19958        nested.insert(
19959            "value".to_string(),
19960            AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
19961        );
19962        let values = vec![
19963            AvroValue::Bytes(vec![0, 255]),
19964            AvroValue::Long(1),
19965            AvroValue::Double(1.0),
19966            AvroValue::Array(Vec::new()),
19967            AvroValue::Map(std::mem::take(&mut empty_map)),
19968            AvroValue::Map(nested),
19969        ];
19970        let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
19971        let transport = vec![json!({
19972            "schema": MESSAGE_STREAM_SCHEMA,
19973            "stream_name": "orders",
19974            "message_id": "message-1",
19975            "position": 1,
19976            "payload_envelope": payload,
19977        })];
19978
19979        for _ in 0..2 {
19980            let Some(MessageStreamDelivery::Message(message)) =
19981                decode_message_stream_delivery(transport.clone()).expect("delivery")
19982            else {
19983                panic!("message delivery expected");
19984            };
19985            assert_eq!(message.arguments, values);
19986            assert!(matches!(message.arguments[1], AvroValue::Long(1)));
19987            assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
19988        }
19989    }
19990
19991    #[test]
19992    fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
19993        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19994            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19995                value.to_string(),
19996            )]))
19997            .expect("message payload");
19998            json!({
19999                "schema": MESSAGE_STREAM_SCHEMA,
20000                "stream_name": "orders",
20001                "message_id": message_id,
20002                "position": position,
20003                "payload_envelope": payload,
20004            })
20005        }
20006
20007        fn opened(sequence: u64) -> HistoryEvent {
20008            history_event(
20009                "SignalWaitOpened",
20010                json!({
20011                    "sequence": sequence,
20012                    "signal_name": MESSAGE_STREAM_SIGNAL,
20013                }),
20014            )
20015        }
20016
20017        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20018            history_event(
20019                "SignalApplied",
20020                json!({
20021                    "sequence": sequence,
20022                    "signal_name": MESSAGE_STREAM_SIGNAL,
20023                    "value": fixture_envelope(json!([delivery])),
20024                }),
20025            )
20026        }
20027
20028        fn worker() -> Worker {
20029            let client = Client::new("http://127.0.0.1:8080").expect("client");
20030            let mut worker = Worker::new(client, "rust-workers");
20031            worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20032                let stream = ctx.message_stream("orders")?;
20033                let first = stream.receive_one().await?;
20034                let second = stream.receive_one().await?;
20035                Ok(json!([first.message_id, second.message_id]))
20036            });
20037            worker
20038        }
20039
20040        fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20041            let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20042            task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20043            task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20044            task
20045        }
20046
20047        let waiting = worker()
20048            .execute_workflow_task_decision(workflow_task(
20049                "rust.message-stream",
20050                Vec::new(),
20051                DEFAULT_CODEC,
20052            ))
20053            .expect("first worker opens the stream wait");
20054        assert_eq!(
20055            waiting.commands,
20056            vec![json!({
20057                "type": "open_signal_wait",
20058                "signal_name": MESSAGE_STREAM_SIGNAL,
20059            })]
20060        );
20061        assert!(waiting.message_stream_cursors.is_empty());
20062        assert_eq!(
20063            waiting.message_stream_waits,
20064            vec![json!({"stream_name": "orders", "after_position": 0})]
20065        );
20066
20067        let first_delivery = delivery("message-1", 1, "one");
20068        let first_arrival = worker()
20069            .execute_workflow_task_decision(task_with_resume(
20070                vec![opened(1)],
20071                first_delivery.clone(),
20072            ))
20073            .expect("replacement worker consumes the first arrival");
20074        assert_eq!(
20075            first_arrival.commands,
20076            vec![json!({
20077                "type": "open_signal_wait",
20078                "signal_name": MESSAGE_STREAM_SIGNAL,
20079            })]
20080        );
20081        assert_eq!(
20082            first_arrival.message_stream_cursors,
20083            vec![json!({"stream_name": "orders", "through_position": 1})]
20084        );
20085        assert_eq!(
20086            first_arrival.message_stream_waits,
20087            vec![json!({"stream_name": "orders", "after_position": 1})]
20088        );
20089
20090        let second_delivery = delivery("message-2", 2, "two");
20091        let first_applied = applied(1, first_delivery);
20092        let completed = worker()
20093            .execute_workflow_task_decision(task_with_resume(
20094                vec![opened(1), first_applied.clone(), opened(2)],
20095                second_delivery.clone(),
20096            ))
20097            .expect("next replacement worker consumes the second arrival");
20098        assert_eq!(completed.commands.len(), 1);
20099        assert_eq!(completed.commands[0]["type"], "complete_workflow");
20100        assert_eq!(
20101            decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20102                .expect("workflow result"),
20103            json!(["message-1", "message-2"])
20104        );
20105        assert_eq!(
20106            completed.message_stream_cursors,
20107            vec![json!({"stream_name": "orders", "through_position": 2})]
20108        );
20109        assert!(completed.message_stream_waits.is_empty());
20110
20111        let replay_history = vec![
20112            opened(1),
20113            first_applied,
20114            opened(2),
20115            applied(2, second_delivery),
20116        ];
20117        for _cold_worker_or_restart in 0..2 {
20118            let replayed = worker()
20119                .execute_workflow_task_decision(workflow_task(
20120                    "rust.message-stream",
20121                    replay_history.clone(),
20122                    DEFAULT_CODEC,
20123                ))
20124                .expect("cold worker replays each logical message exactly once");
20125            assert_eq!(replayed.commands.len(), 1);
20126            assert_eq!(
20127                decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20128                    .expect("replayed workflow result"),
20129                json!(["message-1", "message-2"])
20130            );
20131            assert_eq!(
20132                replayed.message_stream_cursors,
20133                vec![json!({"stream_name": "orders", "through_position": 2})]
20134            );
20135            assert!(replayed.message_stream_waits.is_empty());
20136        }
20137    }
20138
20139    #[test]
20140    fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20141        assert!(!worker_protocol_supports_message_streams("1.14"));
20142        assert!(worker_protocol_supports_message_streams("1.15"));
20143        assert!(worker_protocol_supports_message_streams("1.16"));
20144        assert!(worker_protocol_supports_message_streams(
20145            WORKER_PROTOCOL_VERSION
20146        ));
20147        assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20148    }
20149
20150    #[test]
20151    fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20152        let ctx = workflow_context(vec![
20153            history_event(
20154                "ConditionWaitOpened",
20155                json!({
20156                    "sequence": 1,
20157                    "condition_wait_id": "condition:1",
20158                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20159                    "condition_key": "signal:finish",
20160                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20161                }),
20162            ),
20163            history_event(
20164                "ConditionWaitSatisfied",
20165                json!({
20166                    "sequence": 1,
20167                    "condition_wait_id": "condition:1",
20168                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20169                    "condition_key": "signal:finish",
20170                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20171                }),
20172            ),
20173            history_event(
20174                "SignalReceived",
20175                json!({"signal_name": "finish", "arguments": []}),
20176            ),
20177        ]);
20178        let mut signal = Box::pin(ctx.wait_signal("finish"));
20179        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20180
20181        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20182            signal.as_mut().poll(&mut task_context)
20183        else {
20184            panic!("condition history must not resolve as a typed signal wait");
20185        };
20186        assert_eq!(failure.reason, "recorded_command_mismatch");
20187        assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20188    }
20189
20190    #[test]
20191    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20192        let signal_then_timer = vec![
20193            history_event(
20194                "SignalWaitOpened",
20195                json!({"sequence": 1, "signal_name": "go"}),
20196            ),
20197            history_event(
20198                "SignalApplied",
20199                json!({
20200                    "sequence": 1,
20201                    "signal_name": "go",
20202                    "value": fixture_envelope(json!(["now"])),
20203                }),
20204            ),
20205            history_event(
20206                "TimerScheduled",
20207                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20208            ),
20209            history_event(
20210                "TimerFired",
20211                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20212            ),
20213        ];
20214
20215        let ctx = workflow_context(signal_then_timer.clone());
20216        let mut signal = Box::pin(ctx.wait_signal("go"));
20217        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20218        assert!(matches!(
20219            signal.as_mut().poll(&mut task_context),
20220            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20221        ));
20222        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20223        assert!(matches!(
20224            timer.as_mut().poll(&mut task_context),
20225            Poll::Ready(Ok(()))
20226        ));
20227        ctx.ensure_history_consumed()
20228            .expect("signal and timer history consumed in order");
20229
20230        let reordered = workflow_context(signal_then_timer);
20231        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20232        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20233            timer_first.as_mut().poll(&mut task_context)
20234        else {
20235            panic!("timer cannot consume signal-wait-first history");
20236        };
20237        assert_eq!(failure.reason, "recorded_command_mismatch");
20238        assert_eq!(failure.sequence, Some(1));
20239        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20240
20241        let timer_then_signal = vec![
20242            history_event(
20243                "TimerScheduled",
20244                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20245            ),
20246            history_event(
20247                "TimerFired",
20248                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20249            ),
20250            history_event(
20251                "SignalWaitOpened",
20252                json!({"sequence": 2, "signal_name": "go"}),
20253            ),
20254            history_event(
20255                "SignalApplied",
20256                json!({
20257                    "sequence": 2,
20258                    "signal_name": "go",
20259                    "value": fixture_envelope(json!([])),
20260                }),
20261            ),
20262        ];
20263        let reordered = workflow_context(timer_then_signal);
20264        let mut signal_first = Box::pin(reordered.wait_signal("go"));
20265        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20266            signal_first.as_mut().poll(&mut task_context)
20267        else {
20268            panic!("signal wait cannot consume timer-first history");
20269        };
20270        assert_eq!(failure.reason, "recorded_command_mismatch");
20271        assert_eq!(failure.sequence, Some(1));
20272        assert_eq!(failure.expected.as_deref(), Some("timer"));
20273    }
20274
20275    #[test]
20276    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20277        let duplicate_timer = WorkflowState::new(
20278            vec![
20279                history_event(
20280                    "TimerScheduled",
20281                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20282                ),
20283                history_event(
20284                    "TimerScheduled",
20285                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20286                ),
20287            ],
20288            "rust-workers".to_string(),
20289            DEFAULT_CODEC.to_string(),
20290            None,
20291        )
20292        .expect_err("one workflow sequence cannot schedule two timers");
20293        assert!(matches!(
20294            duplicate_timer,
20295            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20296                if reason == "timer_schedule_missing_or_duplicate"
20297        ));
20298
20299        let colliding_kinds = WorkflowState::new(
20300            vec![
20301                history_event(
20302                    "TimerScheduled",
20303                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20304                ),
20305                history_event(
20306                    "ActivityCompleted",
20307                    json!({"sequence": 1, "activity_type": "same-sequence"}),
20308                ),
20309            ],
20310            "rust-workers".to_string(),
20311            DEFAULT_CODEC.to_string(),
20312            None,
20313        )
20314        .expect_err("one workflow sequence cannot identify two command kinds");
20315        assert!(matches!(
20316            colliding_kinds,
20317            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20318                if reason == "durable_command_sequence_collision"
20319        ));
20320
20321        let duplicate_signal_wait = WorkflowState::new(
20322            vec![
20323                history_event(
20324                    "SignalWaitOpened",
20325                    json!({"sequence": 1, "signal_name": "go"}),
20326                ),
20327                history_event(
20328                    "SignalWaitOpened",
20329                    json!({"sequence": 1, "signal_name": "go"}),
20330                ),
20331            ],
20332            "rust-workers".to_string(),
20333            DEFAULT_CODEC.to_string(),
20334            None,
20335        )
20336        .expect_err("one workflow sequence cannot open two signal waits");
20337        assert!(matches!(
20338            duplicate_signal_wait,
20339            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20340                if reason == "signal_wait_open_missing_or_duplicate"
20341        ));
20342    }
20343
20344    #[test]
20345    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20346        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20347            .expect("side-effect result");
20348        let ctx = workflow_context(vec![history_event(
20349            "SideEffectRecorded",
20350            json!({"sequence": 99, "result": result}),
20351        )]);
20352
20353        let replayed: Value = ctx
20354            .side_effect(|| panic!("recorded side effect must not run"))
20355            .expect("positive global workflow sequence is valid");
20356        assert_eq!(replayed, json!({"captured": true}));
20357        ctx.ensure_history_consumed().expect("history consumed");
20358    }
20359
20360    #[test]
20361    fn workflow_history_rejects_zero_and_descending_command_sequences() {
20362        let result =
20363            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20364        let zero = WorkflowState::new(
20365            vec![history_event(
20366                "SideEffectRecorded",
20367                json!({"sequence": 0, "result": result.clone()}),
20368            )],
20369            "rust-workers".to_string(),
20370            DEFAULT_CODEC.to_string(),
20371            None,
20372        )
20373        .expect_err("durable command sequences must be positive");
20374        assert!(matches!(
20375            zero,
20376            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20377                if reason == "durable_command_sequence_invalid"
20378        ));
20379
20380        let descending = WorkflowState::new(
20381            vec![
20382                history_event(
20383                    "SideEffectRecorded",
20384                    json!({"sequence": 3, "result": result}),
20385                ),
20386                history_event(
20387                    "VersionMarkerRecorded",
20388                    json!({
20389                        "sequence": 2,
20390                        "change_id": "descending-marker",
20391                        "version": 1,
20392                        "min_supported": 1,
20393                        "max_supported": 1,
20394                    }),
20395                ),
20396            ],
20397            "rust-workers".to_string(),
20398            DEFAULT_CODEC.to_string(),
20399            None,
20400        )
20401        .expect_err("new durable commands must remain strictly ordered");
20402        let Error::NonDeterministicReplay(failure) = descending else {
20403            panic!("expected typed replay failure");
20404        };
20405        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20406        assert_eq!(failure.sequence, Some(2));
20407        assert_eq!(
20408            failure.expected.as_deref(),
20409            Some("workflow sequence greater than 3")
20410        );
20411        assert_eq!(failure.actual.as_deref(), Some("2"));
20412    }
20413
20414    #[test]
20415    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20416        fn worker() -> Worker {
20417            let client = Client::new("http://127.0.0.1:8080").expect("client");
20418            let mut worker = Worker::new(client, "rust-workers");
20419            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20420                ctx.wait_signal("finish").await?;
20421                let marker: String =
20422                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20423                assert_eq!(marker, "after-finish");
20424                Ok(json!("finished"))
20425            });
20426            worker
20427        }
20428
20429        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20430            .expect("side-effect result");
20431        let task = workflow_task(
20432            "rust.finish-after-gaps",
20433            vec![
20434                history_event(
20435                    "SignalWaitOpened",
20436                    json!({"sequence": 1, "signal_name": "finish"}),
20437                ),
20438                history_event(
20439                    "SignalReceived",
20440                    json!({
20441                        "signal_id": "increment-3",
20442                        "signal_name": "increment",
20443                        "workflow_sequence": 2,
20444                        "payload_codec": DEFAULT_CODEC,
20445                        "arguments": fixture_envelope(json!([3])),
20446                    }),
20447                ),
20448                history_event(
20449                    "SignalReceived",
20450                    json!({
20451                        "signal_id": "increment-5",
20452                        "signal_name": "increment",
20453                        "workflow_sequence": 3,
20454                        "payload_codec": DEFAULT_CODEC,
20455                        "arguments": fixture_envelope(json!([5])),
20456                    }),
20457                ),
20458                history_event(
20459                    "SignalReceived",
20460                    json!({
20461                        "signal_id": "finish",
20462                        "signal_name": "finish",
20463                        "workflow_sequence": 4,
20464                        "payload_codec": DEFAULT_CODEC,
20465                        "arguments": fixture_envelope(json!([])),
20466                    }),
20467                ),
20468                history_event(
20469                    "SignalApplied",
20470                    json!({
20471                        "sequence": 1,
20472                        "signal_id": "finish",
20473                        "signal_name": "finish",
20474                        "payload_codec": DEFAULT_CODEC,
20475                        "value": fixture_envelope(json!([])),
20476                    }),
20477                ),
20478                history_event(
20479                    "SideEffectRecorded",
20480                    json!({"sequence": 5, "result": marker}),
20481                ),
20482            ],
20483            DEFAULT_CODEC,
20484        );
20485
20486        for _original_or_cold_worker in 0..2 {
20487            let commands = worker()
20488                .execute_workflow_task(task.clone())
20489                .expect("signal gaps preserve deterministic replay");
20490            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20491            assert_eq!(commands[0]["type"], "complete_workflow");
20492            assert_eq!(
20493                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20494                json!("finished")
20495            );
20496        }
20497    }
20498
20499    #[test]
20500    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20501        let ctx = workflow_context(Vec::new());
20502        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20503        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20504        assert!(matches!(
20505            sleep.as_mut().poll(&mut task_context),
20506            Poll::Ready(Err(Error::TimerDurationOverflow))
20507        ));
20508        assert!(ctx.take_commands().expect("commands").is_empty());
20509    }
20510
20511    #[test]
20512    fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20513        let entries = AvroValue::Map(BTreeMap::from([
20514            ("text".to_string(), AvroValue::String("same".to_string())),
20515            (
20516                "nested".to_string(),
20517                AvroValue::Map(BTreeMap::from([
20518                    ("beta".to_string(), AvroValue::Long(2)),
20519                    ("alpha".to_string(), AvroValue::Long(1)),
20520                ])),
20521            ),
20522            ("long".to_string(), AvroValue::Long(7)),
20523            ("double".to_string(), AvroValue::Double(7.0)),
20524            ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20525        ]));
20526        let ctx = workflow_context(Vec::new());
20527        ctx.upsert_memo(entries.clone()).expect("valid memo update");
20528        let commands = ctx.take_commands().expect("commands");
20529
20530        assert_eq!(commands.len(), 1);
20531        assert_eq!(commands[0]["type"], "upsert_memo");
20532        let server_entries = json!({
20533            "codec": "avro",
20534            "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20535        });
20536        assert_eq!(
20537            commands[0]["entries"]
20538                .as_object()
20539                .expect("entries envelope")
20540                .keys()
20541                .collect::<Vec<_>>(),
20542            vec!["blob", "codec"]
20543        );
20544        assert_eq!(commands[0]["entries"], server_entries);
20545        let wire_entries =
20546            decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20547        assert_eq!(wire_entries, entries);
20548
20549        let history = vec![history_event(
20550            "MemoUpserted",
20551            json!({
20552                "sequence": 1,
20553                "entries": server_entries.clone(),
20554                "merged": server_entries,
20555            }),
20556        )];
20557        let replay = workflow_context(history.clone());
20558        replay
20559            .upsert_memo(entries.clone())
20560            .expect("matching replay identity");
20561        assert!(replay.take_commands().expect("replay commands").is_empty());
20562
20563        let changed_types = AvroValue::Map(BTreeMap::from([
20564            ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20565            (
20566                "nested".to_string(),
20567                AvroValue::Map(BTreeMap::from([
20568                    ("alpha".to_string(), AvroValue::Long(1)),
20569                    ("beta".to_string(), AvroValue::Long(2)),
20570                ])),
20571            ),
20572            ("long".to_string(), AvroValue::Double(7.0)),
20573            ("double".to_string(), AvroValue::Long(7)),
20574            ("binary".to_string(), AvroValue::String("same".to_string())),
20575        ]));
20576        let error = workflow_context(history)
20577            .upsert_memo(changed_types)
20578            .expect_err("memo replay identity must preserve Avro value types");
20579        assert!(matches!(
20580            error,
20581            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20582        ));
20583    }
20584
20585    #[test]
20586    fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20587        let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20588            .expect("memo envelope");
20589        let replay = workflow_context(vec![history_event(
20590            "MemoUpserted",
20591            json!({
20592                "sequence": 1,
20593                "entries": original.clone(),
20594                "merged": original
20595            }),
20596        )]);
20597        let error = replay
20598            .upsert_memo(json!({"stage": "changed"}))
20599            .expect_err("changed memo update must fail replay");
20600        assert!(matches!(
20601            error,
20602            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20603        ));
20604
20605        let invalid = workflow_context(Vec::new())
20606            .upsert_memo(
20607                json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20608            )
20609            .expect_err("oversized key");
20610        assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20611    }
20612
20613    #[test]
20614    fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20615        let negative_zero = AvroValue::Map(BTreeMap::from([(
20616            "reading".to_string(),
20617            AvroValue::Double(-0.0),
20618        )]));
20619        let negative_zero_envelope =
20620            encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20621        let history = vec![history_event(
20622            "MemoUpserted",
20623            json!({
20624                "sequence": 1,
20625                "entries": negative_zero_envelope.clone(),
20626                "merged": negative_zero_envelope,
20627            }),
20628        )];
20629
20630        workflow_context(history.clone())
20631            .upsert_memo(negative_zero)
20632            .expect("matching negative-zero history identity");
20633
20634        let error = workflow_context(history)
20635            .upsert_memo(AvroValue::Map(BTreeMap::from([(
20636                "reading".to_string(),
20637                AvroValue::Double(0.0),
20638            )])))
20639            .expect_err("positive zero must not consume negative-zero memo history");
20640        assert!(matches!(
20641            error,
20642            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20643        ));
20644    }
20645
20646    #[test]
20647    fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20648        let supported = json!({
20649            "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20650            "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20651        });
20652        assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20653        assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20654            "workflow_memo_updates": {"supported": false},
20655            "supported_workflow_task_commands": ["upsert_memo"]
20656        }))));
20657        assert!(commands_use_workflow_memo_updates(&[json!({
20658            "type": "upsert_memo",
20659            "entries": {"stage": "processing"}
20660        })]));
20661    }
20662
20663    #[test]
20664    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20665        let client = Client::new("http://127.0.0.1:8080").expect("client");
20666        let mut worker = Worker::new(client, "rust-workers");
20667        worker.register_workflow("rust.timer", |ctx, _input| async move {
20668            ctx.sleep(Duration::from_secs(5)).await?;
20669            ctx.activity("after-timer", json!([])).await
20670        });
20671
20672        let task = |history_events| WorkflowTask {
20673            task_id: "wft-rust-timer-1".to_string(),
20674            workflow_command_id: None,
20675            workflow_id: Some("wf-rust-timer".to_string()),
20676            run_id: Some("run-rust-timer".to_string()),
20677            workflow_type: "rust.timer".to_string(),
20678            cancel_requested: false,
20679            payload_codec: DEFAULT_CODEC.to_string(),
20680            arguments: Some(
20681                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20682            ),
20683            history_events,
20684            total_history_events: None,
20685            history_size_bytes: None,
20686            continue_as_new_recommended: None,
20687            history_budget_pressure: None,
20688            next_history_page_token: None,
20689            workflow_task_attempt: 1,
20690            workflow_signal_id: None,
20691            signal_name: None,
20692            signal_arguments: None,
20693            workflow_update_id: None,
20694            update_name: None,
20695            lease_owner: Some("rust-worker".to_string()),
20696        };
20697
20698        let initial = worker
20699            .execute_workflow_task(task(Vec::new()))
20700            .expect("initial timer task");
20701        assert_eq!(
20702            initial,
20703            vec![json!({"type": "start_timer", "delay_seconds": 5})]
20704        );
20705
20706        let activity_result =
20707            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20708        let replayed = worker
20709            .execute_workflow_task(task(vec![
20710                history_event(
20711                    "TimerScheduled",
20712                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20713                ),
20714                history_event(
20715                    "TimerFired",
20716                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20717                ),
20718                history_event(
20719                    "ActivityCompleted",
20720                    json!({
20721                        "sequence": 2,
20722                        "activity_type": "after-timer",
20723                        "payload_codec": DEFAULT_CODEC,
20724                        "result": activity_result,
20725                    }),
20726                ),
20727            ]))
20728            .expect("replayed workflow task");
20729        assert_eq!(replayed.len(), 1);
20730        assert_eq!(replayed[0]["type"], "complete_workflow");
20731        assert_eq!(
20732            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20733            json!("done")
20734        );
20735    }
20736
20737    #[test]
20738    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20739        let client = Client::new("http://127.0.0.1:8080").expect("client");
20740        let mut worker = Worker::new(client, "rust-workers");
20741        worker.register_workflow("rust.continue", |ctx, _input| async move {
20742            ctx.continue_as_new_with_options(
20743                ContinueAsNewOptions::new()
20744                    .workflow_type("rust.next")
20745                    .task_queue("next-workers"),
20746                json!([2, {"cursor": "next"}]),
20747            )
20748        });
20749
20750        let commands = worker
20751            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20752            .expect("continue-as-new command");
20753
20754        assert_eq!(commands.len(), 1);
20755        assert_eq!(commands[0]["type"], "continue_as_new");
20756        assert_eq!(commands[0]["workflow_type"], "rust.next");
20757        assert_eq!(commands[0]["queue"], "next-workers");
20758        assert_eq!(
20759            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20760                .expect("continue-as-new arguments"),
20761            json!([2, {"cursor": "next"}])
20762        );
20763    }
20764
20765    #[test]
20766    fn continue_as_new_preserves_typed_arguments() {
20767        let client = Client::new("http://127.0.0.1:8080").expect("client");
20768        let mut worker = Worker::new(client, "rust-workers");
20769        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20770            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20771            unreachable!("continue-as-new returns a control-flow error")
20772        });
20773
20774        let commands = worker
20775            .execute_workflow_task(workflow_task(
20776                "rust.typed-continue",
20777                Vec::new(),
20778                DEFAULT_CODEC,
20779            ))
20780            .expect("typed continue-as-new command");
20781
20782        assert_eq!(commands[0]["type"], "continue_as_new");
20783        assert_eq!(
20784            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20785                .expect("typed continue arguments"),
20786            AvroValue::Array(vec![typed_fidelity_probe()])
20787        );
20788    }
20789
20790    #[test]
20791    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20792        let client = Client::new("http://127.0.0.1:8080").expect("client");
20793        let mut worker = Worker::new(client, "rust-workers");
20794        worker.register_workflow("rust.continue", |ctx, _input| async move {
20795            ctx.continue_as_new(json!([2]))
20796        });
20797        let task = workflow_task(
20798            "rust.continue",
20799            vec![history_event(
20800                "WorkflowContinuedAsNew",
20801                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20802            )],
20803            DEFAULT_CODEC,
20804        );
20805
20806        for _worker_restart_or_redelivery in 0..2 {
20807            let commands = worker
20808                .execute_workflow_task(task.clone())
20809                .expect("recorded transition replays");
20810            assert!(
20811                commands.is_empty(),
20812                "replay must not emit another successor"
20813            );
20814        }
20815    }
20816
20817    #[test]
20818    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20819        let ctx = workflow_context(Vec::new());
20820        let error = ctx
20821            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
20822            .expect_err("blank queue must be rejected");
20823
20824        let Error::InvalidContinueAsNewOptions(error) = error else {
20825            panic!("expected typed continue-as-new validation error");
20826        };
20827        assert_eq!(error.field, "task_queue");
20828        assert!(ctx.take_commands().expect("commands").is_empty());
20829    }
20830
20831    #[test]
20832    fn workflow_context_exposes_server_history_budget() {
20833        let client = Client::new("http://127.0.0.1:8080").expect("client");
20834        let mut worker = Worker::new(client, "rust-workers");
20835        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20836            let budget = ctx.history_budget()?;
20837            Ok(json!({
20838                "events": budget.event_count,
20839                "bytes": budget.size_bytes,
20840                "recommended": budget.continue_as_new_recommended,
20841                "pressure": budget.pressure,
20842            }))
20843        });
20844        let task: WorkflowTask = serde_json::from_value(json!({
20845            "task_id": "task-history-budget",
20846            "workflow_type": "rust.history-budget",
20847            "payload_codec": DEFAULT_CODEC,
20848            "history_events": [],
20849            "total_history_events": 480,
20850            "history_size_bytes": 1_048_576,
20851            "continue_as_new_recommended": true,
20852            "history_budget_pressure": "continue_as_new_recommended",
20853        }))
20854        .expect("published workflow task");
20855
20856        let commands = worker
20857            .execute_workflow_task(task)
20858            .expect("history-budget workflow");
20859        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20860        assert_eq!(result["events"], 480);
20861        assert_eq!(result["bytes"], 1_048_576);
20862        assert_eq!(result["recommended"], true);
20863        assert_eq!(result["pressure"], "continue_as_new_recommended");
20864    }
20865
20866    #[test]
20867    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20868        let client = Client::new("http://127.0.0.1:8080").expect("client");
20869        let mut worker = Worker::new(client, "rust-workers");
20870        worker.register_workflow("rust.failing", |_ctx, _input| async move {
20871            Err(Error::Codec("rust_conformance_failure".to_string()))
20872        });
20873        let task = WorkflowTask {
20874            task_id: "wft-rust-failing-1".to_string(),
20875            workflow_command_id: None,
20876            workflow_id: Some("wf-rust-failing".to_string()),
20877            run_id: Some("run-rust-failing".to_string()),
20878            workflow_type: "rust.failing".to_string(),
20879            cancel_requested: false,
20880            payload_codec: DEFAULT_CODEC.to_string(),
20881            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20882            history_events: Vec::new(),
20883            total_history_events: Some(0),
20884            history_size_bytes: None,
20885            continue_as_new_recommended: None,
20886            history_budget_pressure: None,
20887            next_history_page_token: None,
20888            workflow_task_attempt: 1,
20889            workflow_signal_id: None,
20890            signal_name: None,
20891            signal_arguments: None,
20892            workflow_update_id: None,
20893            update_name: None,
20894            lease_owner: Some("rust-worker".to_string()),
20895        };
20896
20897        let commands = worker
20898            .execute_workflow_task(task)
20899            .expect("handler failure becomes a workflow command");
20900
20901        assert_eq!(commands.len(), 1);
20902        assert_eq!(commands[0]["type"], "fail_workflow");
20903        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20904        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20905        assert_eq!(commands[0]["non_retryable"], false);
20906        assert_eq!(
20907            commands[0]["message"],
20908            "codec error: rust_conformance_failure"
20909        );
20910        assert_eq!(
20911            commands[0]["exception"]["message"],
20912            "codec error: rust_conformance_failure"
20913        );
20914    }
20915
20916    #[test]
20917    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
20918        let client = Client::new("http://127.0.0.1:8080").expect("client");
20919        let mut worker = Worker::new(client, "rust-workers");
20920        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
20921            let _: String = ctx.side_effect(|| "captured".to_string())?;
20922            Err(Error::WorkerLoop("application failure".to_string()))
20923        });
20924
20925        let commands = worker
20926            .execute_workflow_task(workflow_task(
20927                "rust.failing-after-side-effect",
20928                Vec::new(),
20929                DEFAULT_CODEC,
20930            ))
20931            .expect("ordinary failure remains a workflow decision");
20932
20933        assert_eq!(commands.len(), 2);
20934        assert_eq!(commands[0]["type"], "record_side_effect");
20935        assert_eq!(commands[1]["type"], "fail_workflow");
20936    }
20937
20938    #[test]
20939    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
20940        let client = Client::new("http://127.0.0.1:8080").expect("client");
20941        let mut worker = Worker::new(client, "rust-workers");
20942        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
20943            Err(Error::WorkerLoop("application failure".to_string()))
20944        });
20945        let result =
20946            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
20947
20948        let error = worker
20949            .execute_workflow_task(workflow_task(
20950                "rust.removed-side-effect",
20951                vec![history_event(
20952                    "SideEffectRecorded",
20953                    json!({"sequence": 1, "result": result}),
20954                )],
20955                DEFAULT_CODEC,
20956            ))
20957            .expect_err("removed committed history must not become fail_workflow");
20958
20959        let Error::NonDeterministicReplay(failure) = error else {
20960            panic!("expected typed replay failure");
20961        };
20962        assert_eq!(failure.reason, "recorded_commands_unconsumed");
20963        assert_eq!(failure.sequence, Some(1));
20964        assert_eq!(failure.expected.as_deref(), Some("side effect"));
20965    }
20966
20967    #[test]
20968    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
20969        let client = Client::new("http://127.0.0.1:8080").expect("client");
20970        let mut worker = Worker::new(client, "rust-workers");
20971        worker.register_workflow(
20972            "rust.side-effect-before-marker-error",
20973            |ctx, _input| async move {
20974                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
20975                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
20976                ctx.get_version("restart-safe", 2, 2)?;
20977                Ok(Value::Null)
20978            },
20979        );
20980
20981        let error = worker
20982            .execute_workflow_task(workflow_task(
20983                "rust.side-effect-before-marker-error",
20984                vec![history_event(
20985                    "VersionMarkerRecorded",
20986                    json!({
20987                        "sequence": 1,
20988                        "change_id": "restart-safe",
20989                        "version": 1,
20990                        "min_supported": 1,
20991                        "max_supported": 1,
20992                    }),
20993                )],
20994                DEFAULT_CODEC,
20995            ))
20996            .expect_err("replay error must return no queued workflow commands");
20997
20998        let Error::NonDeterministicReplay(failure) = error else {
20999            panic!("expected typed replay failure");
21000        };
21001        assert_eq!(failure.reason, "version_marker_incompatible_range");
21002        assert_eq!(failure.sequence, Some(1));
21003    }
21004
21005    #[test]
21006    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
21007        let client = Client::new("http://127.0.0.1:8080").expect("client");
21008        let mut worker = Worker::new(client, "rust-workers");
21009        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
21010            ctx.sleep(Duration::from_secs(5)).await?;
21011            Ok(json!({"status": "timer fired"}))
21012        });
21013
21014        let task = WorkflowTask {
21015            task_id: "wft-rust-timer-pending".to_string(),
21016            workflow_command_id: None,
21017            workflow_id: Some("wf-rust-timer".to_string()),
21018            run_id: Some("run-rust-timer".to_string()),
21019            workflow_type: "rust.timer.pending".to_string(),
21020            cancel_requested: false,
21021            payload_codec: DEFAULT_CODEC.to_string(),
21022            arguments: Some(
21023                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21024            ),
21025            history_events: vec![history_event(
21026                "TimerScheduled",
21027                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21028            )],
21029            total_history_events: Some(1),
21030            history_size_bytes: None,
21031            continue_as_new_recommended: None,
21032            history_budget_pressure: None,
21033            next_history_page_token: None,
21034            workflow_task_attempt: 1,
21035            workflow_signal_id: None,
21036            signal_name: None,
21037            signal_arguments: None,
21038            workflow_update_id: None,
21039            update_name: None,
21040            lease_owner: Some("rust-worker".to_string()),
21041        };
21042
21043        for _redelivery_or_restart in 0..2 {
21044            let commands = worker
21045                .execute_workflow_task(task.clone())
21046                .expect("recorded timer remains pending");
21047            assert!(
21048                commands.is_empty(),
21049                "recorded timer must not be rescheduled"
21050            );
21051        }
21052    }
21053
21054    #[test]
21055    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21056        let client = Client::new("http://127.0.0.1:8080").expect("client");
21057        let mut worker = Worker::new(client, "rust-workers");
21058        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21059            Ok(json!({"status": "completed"}))
21060        });
21061        let task = WorkflowTask {
21062            task_id: "wft-rust-timer-removed".to_string(),
21063            workflow_command_id: None,
21064            workflow_id: Some("wf-rust-timer".to_string()),
21065            run_id: Some("run-rust-timer".to_string()),
21066            workflow_type: "rust.timer.removed".to_string(),
21067            cancel_requested: false,
21068            payload_codec: DEFAULT_CODEC.to_string(),
21069            arguments: Some(
21070                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21071            ),
21072            history_events: vec![
21073                history_event(
21074                    "TimerScheduled",
21075                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21076                ),
21077                history_event(
21078                    "TimerFired",
21079                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21080                ),
21081            ],
21082            total_history_events: Some(2),
21083            history_size_bytes: None,
21084            continue_as_new_recommended: None,
21085            history_budget_pressure: None,
21086            next_history_page_token: None,
21087            workflow_task_attempt: 1,
21088            workflow_signal_id: None,
21089            signal_name: None,
21090            signal_arguments: None,
21091            workflow_update_id: None,
21092            update_name: None,
21093            lease_owner: Some("rust-worker".to_string()),
21094        };
21095
21096        let Error::NonDeterministicReplay(failure) = worker
21097            .execute_workflow_task(task)
21098            .expect_err("removed timer must fail replay")
21099        else {
21100            panic!("expected typed replay failure");
21101        };
21102        assert_eq!(failure.reason, "recorded_commands_unconsumed");
21103        assert_eq!(failure.sequence, Some(1));
21104    }
21105
21106    #[test]
21107    fn workflow_context_emits_explicit_child_workflow_contract() {
21108        let ctx = WorkflowContext {
21109            state: Arc::new(Mutex::new(
21110                WorkflowState::new_with_identity(
21111                    Vec::new(),
21112                    Some("wf-parent".to_string()),
21113                    Some("run-parent".to_string()),
21114                    "parent-workers".to_string(),
21115                    DEFAULT_CODEC.to_string(),
21116                    None,
21117                )
21118                .expect("workflow state"),
21119            )),
21120        };
21121        let options = ChildWorkflowOptions::new("python-workers")
21122            .parent_close_policy(ParentClosePolicy::RequestCancel)
21123            .retry_policy(ChildWorkflowRetryPolicy {
21124                max_attempts: Some(3),
21125                backoff_seconds: vec![1, 5],
21126                non_retryable_error_types: vec!["ValidationError".to_string()],
21127            })
21128            .execution_timeout_seconds(600)
21129            .run_timeout_seconds(120);
21130        let mut call = Box::pin(ctx.start_child_workflow(
21131            "python.fulfil-order",
21132            options,
21133            json!([{"order_id": "order-42"}]),
21134        ));
21135        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21136
21137        assert!(matches!(
21138            call.as_mut().poll(&mut task_context),
21139            Poll::Pending
21140        ));
21141        let commands = ctx.take_commands().expect("commands");
21142        assert_eq!(commands.len(), 1);
21143        let command = &commands[0];
21144        assert_eq!(command["type"], "start_child_workflow");
21145        assert_eq!(command["workflow_type"], "python.fulfil-order");
21146        assert_eq!(command["queue"], "python-workers");
21147        assert_eq!(command["parent_close_policy"], "request_cancel");
21148        assert_eq!(command["retry_policy"]["max_attempts"], 3);
21149        assert_eq!(command["execution_timeout_seconds"], 600);
21150        assert_eq!(command["run_timeout_seconds"], 120);
21151        assert_eq!(
21152            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21153            json!([{"order_id": "order-42"}])
21154        );
21155    }
21156
21157    fn child_parent_worker() -> Worker {
21158        let client = Client::new("http://127.0.0.1:8080").expect("client");
21159        let mut worker = Worker::new(client, "rust-parent-workers");
21160        worker.register_workflow("rust.parent", |ctx, _input| async move {
21161            let child = ctx
21162                .start_child_workflow(
21163                    "python.child",
21164                    ChildWorkflowOptions::new("python-child-workers")
21165                        .parent_close_policy(ParentClosePolicy::Terminate),
21166                    json!([{"codec_probe": [1, true, "rust"]}]),
21167                )
21168                .await?;
21169            Ok(json!({
21170                "parent_workflow_id": child.parent.workflow_id,
21171                "parent_run_id": child.parent.run_id,
21172                "child_workflow_id": child.child.workflow_id,
21173                "child_run_id": child.child.run_id,
21174                "child_workflow_type": child.child_workflow_type,
21175                "result": child.result,
21176            }))
21177        });
21178        worker
21179    }
21180
21181    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21182        WorkflowTask {
21183            task_id: "wft-child-parent".to_string(),
21184            workflow_command_id: None,
21185            workflow_id: Some("wf-parent".to_string()),
21186            run_id: Some("run-parent".to_string()),
21187            workflow_type: "rust.parent".to_string(),
21188            cancel_requested: false,
21189            payload_codec: DEFAULT_CODEC.to_string(),
21190            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21191            history_events: vec![
21192                HistoryEvent {
21193                    event_type: "ChildWorkflowScheduled".to_string(),
21194                    payload: json!({
21195                        "sequence": 1,
21196                        "child_call_id": "call-child",
21197                        "child_workflow_instance_id": "wf-child",
21198                        "child_workflow_run_id": "run-child",
21199                        "child_workflow_type": "python.child",
21200                    }),
21201                    raw: HashMap::new(),
21202                },
21203                HistoryEvent {
21204                    event_type: event_type.to_string(),
21205                    payload,
21206                    raw: HashMap::new(),
21207                },
21208            ],
21209            total_history_events: Some(2),
21210            history_size_bytes: None,
21211            continue_as_new_recommended: None,
21212            history_budget_pressure: None,
21213            next_history_page_token: None,
21214            workflow_task_attempt: 1,
21215            workflow_signal_id: None,
21216            signal_name: None,
21217            signal_arguments: None,
21218            workflow_update_id: None,
21219            update_name: None,
21220            lease_owner: Some("rust-worker".to_string()),
21221        }
21222    }
21223
21224    #[test]
21225    fn committed_child_result_replays_without_starting_a_duplicate() {
21226        let worker = child_parent_worker();
21227        let task = child_parent_task(
21228            "ChildRunCompleted",
21229            json!({
21230                "sequence": 1,
21231                "child_call_id": "call-child",
21232                "child_workflow_instance_id": "wf-child",
21233                "child_workflow_run_id": "run-child",
21234                "child_workflow_type": "python.child",
21235                "payload_codec": DEFAULT_CODEC,
21236                "result": fixture_envelope(json!({"from":"python","ok":true})),
21237            }),
21238        );
21239
21240        for _restart in 0..2 {
21241            let commands = worker
21242                .execute_workflow_task(task.clone())
21243                .expect("replayed parent task");
21244            assert_eq!(commands.len(), 1);
21245            assert_eq!(commands[0]["type"], "complete_workflow");
21246            assert!(!commands
21247                .iter()
21248                .any(|command| command["type"] == "start_child_workflow"));
21249            let output =
21250                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21251            assert_eq!(output["parent_workflow_id"], "wf-parent");
21252            assert_eq!(output["parent_run_id"], "run-parent");
21253            assert_eq!(output["child_workflow_id"], "wf-child");
21254            assert_eq!(output["child_run_id"], "run-child");
21255            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21256        }
21257    }
21258
21259    #[test]
21260    fn typed_child_arguments_and_results_survive_replay() {
21261        let client = Client::new("http://127.0.0.1:8080").expect("client");
21262        let mut worker = Worker::new(client, "rust-parent-workers");
21263        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21264            let child = ctx
21265                .start_child_workflow_avro_value(
21266                    "python.typed-child",
21267                    ChildWorkflowOptions::new("python-workers"),
21268                    AvroValue::Array(vec![typed_fidelity_probe()]),
21269                )
21270                .await?;
21271            Ok(child.result)
21272        });
21273
21274        let initial = worker
21275            .execute_workflow_task(workflow_task(
21276                "rust.typed-parent",
21277                Vec::new(),
21278                DEFAULT_CODEC,
21279            ))
21280            .expect("typed child start");
21281        assert_eq!(initial[0]["type"], "start_child_workflow");
21282        assert_eq!(
21283            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21284                .expect("typed child arguments"),
21285            AvroValue::Array(vec![typed_fidelity_probe()])
21286        );
21287
21288        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21289            .expect("typed child result");
21290        let task = workflow_task(
21291            "rust.typed-parent",
21292            vec![
21293                history_event(
21294                    "ChildWorkflowScheduled",
21295                    json!({
21296                        "sequence": 1,
21297                        "child_call_id": "call-typed",
21298                        "child_workflow_instance_id": "wf-child",
21299                        "child_workflow_run_id": "run-child",
21300                        "child_workflow_type": "python.typed-child",
21301                    }),
21302                ),
21303                history_event(
21304                    "ChildRunCompleted",
21305                    json!({
21306                        "sequence": 1,
21307                        "child_call_id": "call-typed",
21308                        "child_workflow_instance_id": "wf-child",
21309                        "child_workflow_run_id": "run-child",
21310                        "child_workflow_type": "python.typed-child",
21311                        "payload_codec": DEFAULT_CODEC,
21312                        "result": result,
21313                    }),
21314                ),
21315            ],
21316            DEFAULT_CODEC,
21317        );
21318
21319        let commands = worker
21320            .execute_workflow_task(task)
21321            .expect("typed child replay");
21322        assert_eq!(commands[0]["type"], "complete_workflow");
21323        assert_eq!(
21324            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21325                .expect("typed parent result"),
21326            typed_fidelity_probe()
21327        );
21328    }
21329
21330    #[test]
21331    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21332        let worker = child_parent_worker();
21333        let mut task = child_parent_task("unused", Value::Null);
21334        task.history_events.truncate(1);
21335        task.total_history_events = Some(1);
21336
21337        for _redelivery_or_restart in 0..2 {
21338            let commands = worker
21339                .execute_workflow_task(task.clone())
21340                .expect("recorded child remains pending");
21341            assert!(
21342                commands.is_empty(),
21343                "recorded pending child must not be started again"
21344            );
21345        }
21346    }
21347
21348    #[test]
21349    fn child_cancellation_becomes_stable_parent_failure_command() {
21350        let worker = child_parent_worker();
21351        let task = child_parent_task(
21352            "ChildRunCancelled",
21353            json!({
21354                "sequence": 1,
21355                "child_workflow_instance_id": "wf-child",
21356                "child_workflow_run_id": "run-child",
21357                "child_workflow_type": "python.child",
21358                "failure_id": "failure-child",
21359                "failure_category": "cancelled",
21360                "message": "cancelled by parent-close policy",
21361            }),
21362        );
21363
21364        let commands = worker
21365            .execute_workflow_task(task)
21366            .expect("parent settlement");
21367        assert_eq!(commands.len(), 1);
21368        assert_eq!(commands[0]["type"], "fail_workflow");
21369        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21370        assert_eq!(
21371            commands[0]["exception"]["properties"]["reason"],
21372            "cancelled"
21373        );
21374        assert_eq!(
21375            commands[0]["exception"]["properties"]["child_workflow_run_id"],
21376            "run-child"
21377        );
21378    }
21379
21380    #[test]
21381    fn workflow_can_handle_typed_child_failure() {
21382        let client = Client::new("http://127.0.0.1:8080").expect("client");
21383        let mut worker = Worker::new(client, "rust-parent-workers");
21384        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21385            match ctx
21386                .start_child_workflow(
21387                    "python.child",
21388                    ChildWorkflowOptions::new("python-child-workers"),
21389                    json!([]),
21390                )
21391                .await
21392            {
21393                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21394                    "reason": failure.reason,
21395                    "failure_id": failure.failure_id,
21396                    "exception_class": failure.exception_class,
21397                    "child_run_id": failure.child_workflow_run_id,
21398                })),
21399                Err(error) => Err(error),
21400                Ok(_) => Err(Error::WorkerLoop(
21401                    "child unexpectedly succeeded".to_string(),
21402                )),
21403            }
21404        });
21405        let mut task = child_parent_task(
21406            "ChildRunFailed",
21407            json!({
21408                "sequence": 1,
21409                "child_workflow_instance_id": "wf-child",
21410                "child_workflow_run_id": "run-child",
21411                "child_workflow_type": "python.child",
21412                "failure_id": "failure-child",
21413                "failure_category": "child_workflow",
21414                "message": "payment rejected",
21415                "exception": {
21416                    "type": "PaymentRejected",
21417                    "class": "payments.PaymentRejected",
21418                    "message": "payment rejected"
21419                }
21420            }),
21421        );
21422        task.workflow_type = "rust.handled-parent".to_string();
21423
21424        let commands = worker.execute_workflow_task(task).expect("handled failure");
21425        assert_eq!(commands[0]["type"], "complete_workflow");
21426        let output =
21427            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21428        assert_eq!(output["reason"], "child_workflow");
21429        assert_eq!(output["failure_id"], "failure-child");
21430        assert_eq!(output["exception_class"], "payments.PaymentRejected");
21431        assert_eq!(output["child_run_id"], "run-child");
21432    }
21433
21434    #[test]
21435    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21436        let client = Client::new("http://127.0.0.1:8080").expect("client");
21437        let mut worker = Worker::new(client, "rust-workers");
21438
21439        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21440            let signal = ctx.wait_signal("start").await?;
21441            let name = signal
21442                .first()
21443                .and_then(|value| value.as_str())
21444                .unwrap_or("world");
21445            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21446            Ok(json!({
21447                "greeting": greeting,
21448                "language": "rust"
21449            }))
21450        });
21451
21452        let signal_arguments =
21453            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21454        let task = WorkflowTask {
21455            task_id: "wft-rust-signal-1".to_string(),
21456            workflow_command_id: None,
21457            workflow_id: Some("wf-rust-hello".to_string()),
21458            run_id: Some("run-rust-hello".to_string()),
21459            workflow_type: "rust.hello_workflow".to_string(),
21460            cancel_requested: false,
21461            payload_codec: DEFAULT_CODEC.to_string(),
21462            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21463            history_events: vec![HistoryEvent {
21464                event_type: "SignalReceived".to_string(),
21465                payload: json!({
21466                    "signal_id": "sig-rust-1",
21467                    "signal_name": "start"
21468                }),
21469                raw: HashMap::new(),
21470            }],
21471            total_history_events: Some(1),
21472            history_size_bytes: None,
21473            continue_as_new_recommended: None,
21474            history_budget_pressure: None,
21475            next_history_page_token: None,
21476            workflow_task_attempt: 1,
21477            workflow_signal_id: Some("sig-rust-1".to_string()),
21478            signal_name: Some("start".to_string()),
21479            signal_arguments: Some(signal_arguments),
21480            workflow_update_id: None,
21481            update_name: None,
21482            lease_owner: Some("rust-worker".to_string()),
21483        };
21484
21485        let commands = worker.execute_workflow_task(task).expect("workflow task");
21486
21487        assert_eq!(commands.len(), 1);
21488        assert_eq!(commands[0]["type"], "schedule_activity");
21489        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21490        assert_eq!(
21491            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21492            json!(["Rust"])
21493        );
21494    }
21495
21496    #[test]
21497    fn workflow_task_appends_paginated_history_events() {
21498        let mut task = WorkflowTask {
21499            task_id: "wft-rust-pages-1".to_string(),
21500            workflow_command_id: None,
21501            workflow_id: Some("wf-rust-pages".to_string()),
21502            run_id: Some("run-rust-pages".to_string()),
21503            workflow_type: "rust.hello_workflow".to_string(),
21504            cancel_requested: false,
21505            payload_codec: DEFAULT_CODEC.to_string(),
21506            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21507            history_events: vec![HistoryEvent {
21508                event_type: "WorkflowStarted".to_string(),
21509                payload: json!({}),
21510                raw: HashMap::new(),
21511            }],
21512            total_history_events: Some(3),
21513            history_size_bytes: None,
21514            continue_as_new_recommended: None,
21515            history_budget_pressure: None,
21516            next_history_page_token: Some("MQ==".to_string()),
21517            workflow_task_attempt: 1,
21518            workflow_signal_id: None,
21519            signal_name: None,
21520            signal_arguments: None,
21521            workflow_update_id: None,
21522            update_name: None,
21523            lease_owner: Some("rust-worker".to_string()),
21524        };
21525
21526        task.append_history_page(WorkflowTaskHistoryPage {
21527            history_events: vec![
21528                HistoryEvent {
21529                    event_type: "SignalReceived".to_string(),
21530                    payload: json!({
21531                        "signal_id": "sig-rust-1",
21532                        "signal_name": "start",
21533                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21534                            .expect("signal arguments")
21535                    }),
21536                    raw: HashMap::new(),
21537                },
21538                HistoryEvent {
21539                    event_type: "MarkerRecorded".to_string(),
21540                    payload: json!({"sequence": 3}),
21541                    raw: HashMap::new(),
21542                },
21543            ],
21544            total_history_events: Some(3),
21545            next_history_page_token: None,
21546        });
21547
21548        assert_eq!(task.history_events.len(), 3);
21549        assert_eq!(task.total_history_events, Some(3));
21550        assert_eq!(task.next_history_page_token, None);
21551
21552        let signal = task
21553            .history_events
21554            .iter()
21555            .find(|event| event.event_type == "SignalReceived")
21556            .expect("signal event");
21557        assert_eq!(
21558            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21559            vec![AvroValue::String("Rust".to_string())]
21560        );
21561    }
21562
21563    #[tokio::test]
21564    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21565        let client = Client::new("http://127.0.0.1:8080").expect("client");
21566        let mut worker = Worker::new(client, "rust-workers");
21567        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21568        worker.register_query("counter", "current", |ctx, _args| async move {
21569            let mut count = 0_i64;
21570            for signal in ctx.signal_events() {
21571                let value = signal
21572                    .arguments
21573                    .first()
21574                    .and_then(Value::as_i64)
21575                    .unwrap_or_default();
21576                match signal.name.as_str() {
21577                    "increment" => count += value,
21578                    "set" => count = value,
21579                    _ => {}
21580                }
21581            }
21582            Ok(json!(count))
21583        });
21584
21585        let task = QueryTask {
21586            query_task_id: "query-rust-counter".to_string(),
21587            query_task_attempt: 1,
21588            lease_owner: Some("rust-worker".to_string()),
21589            workflow_id: Some("counter-1".to_string()),
21590            run_id: Some("run-counter-1".to_string()),
21591            workflow_type: "counter".to_string(),
21592            query_name: "current".to_string(),
21593            payload_codec: DEFAULT_CODEC.to_string(),
21594            workflow_arguments: Some(
21595                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21596            ),
21597            query_arguments: Some(
21598                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21599            ),
21600            history_events: vec![
21601                HistoryEvent {
21602                    event_type: "SignalReceived".to_string(),
21603                    payload: json!({
21604                        "signal_id": "php-signal-1",
21605                        "signal_name": "increment",
21606                        "workflow_sequence": 1,
21607                        "payload_codec": DEFAULT_CODEC,
21608                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21609                    }),
21610                    raw: HashMap::new(),
21611                },
21612                HistoryEvent {
21613                    event_type: "SignalReceived".to_string(),
21614                    payload: json!({
21615                        "signal_id": "python-signal-2",
21616                        "signal_name": "increment",
21617                        "workflow_sequence": 2,
21618                        "payload_codec": DEFAULT_CODEC,
21619                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21620                    }),
21621                    raw: HashMap::new(),
21622                },
21623                HistoryEvent {
21624                    event_type: "SignalReceived".to_string(),
21625                    payload: json!({
21626                        "signal_id": "rust-signal-3",
21627                        "signal_name": "set",
21628                        "workflow_sequence": 3,
21629                        "payload_codec": DEFAULT_CODEC,
21630                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21631                    }),
21632                    raw: HashMap::new(),
21633                },
21634            ],
21635            history_export: None,
21636            run_status: Some("completed".to_string()),
21637        };
21638
21639        let result = worker.execute_query_task(task).await.expect("query result");
21640        assert_eq!(result.into_json().expect("query projection"), json!(0));
21641    }
21642
21643    #[tokio::test]
21644    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21645        let worker = replay_counter_worker();
21646        let running_history = json!([
21647            {
21648                "type": "ActivityCompleted",
21649                "payload": {
21650                    "sequence": 1,
21651                    "activity_type": "load-counter",
21652                    "payload_codec": DEFAULT_CODEC,
21653                    "result": fixture_envelope(json!("loaded"))
21654                }
21655            },
21656            {
21657                "type": "SignalWaitOpened",
21658                "payload": {
21659                    "sequence": 3,
21660                    "signal_name": "increment"
21661                }
21662            },
21663            {
21664                "type": "SignalReceived",
21665                "payload": {
21666                    "signal_id": "signal-3",
21667                    "signal_name": "increment",
21668                    "workflow_sequence": 2,
21669                    "payload_codec": DEFAULT_CODEC,
21670                    "arguments": fixture_envelope(json!([3]))
21671                }
21672            },
21673            {
21674                "type": "SignalApplied",
21675                "payload": {
21676                    "sequence": 3,
21677                    "signal_id": "signal-3",
21678                    "signal_name": "increment",
21679                    "payload_codec": DEFAULT_CODEC,
21680                    "value": fixture_envelope(json!([3]))
21681                }
21682            }
21683        ]);
21684
21685        let running = worker
21686            .execute_query_task(replay_counter_query(
21687                "current",
21688                running_history.clone(),
21689                "running",
21690            ))
21691            .await
21692            .expect("running replay query");
21693        assert_eq!(
21694            running.clone().into_json().expect("query projection"),
21695            json!({"loaded": "loaded", "count": 3, "finished": false})
21696        );
21697
21698        let detached = worker
21699            .execute_query_task(replay_counter_query(
21700                "detached-mutation",
21701                running_history.clone(),
21702                "running",
21703            ))
21704            .await
21705            .expect("query mutates only its detached state clone");
21706        assert_eq!(detached.into_json().expect("query projection"), json!(999));
21707        let failed = worker
21708            .execute_query_task(replay_counter_query(
21709                "failed-mutation",
21710                running_history.clone(),
21711                "running",
21712            ))
21713            .await
21714            .expect_err("failed query");
21715        assert_eq!(failed.reason, "query_rejected");
21716        let unchanged = worker
21717            .execute_query_task(replay_counter_query("current", running_history, "running"))
21718            .await
21719            .expect("later query reconstructs unchanged state");
21720        assert_eq!(unchanged, running);
21721
21722        let restarted_worker = replay_counter_worker();
21723        let empty_arguments = fixture_envelope(json!([]));
21724        let loaded_result = fixture_envelope(json!("loaded"));
21725        let signal_three = fixture_blob(json!([3]));
21726        let signal_five = fixture_blob(json!([5]));
21727        let restarted_task: QueryTask = serde_json::from_value(json!({
21728            "query_task_id": "query-after-restart",
21729            "workflow_id": "counter-1",
21730            "run_id": "run-counter-1",
21731            "workflow_type": "replay-counter",
21732            "query_name": "current",
21733            "payload_codec": DEFAULT_CODEC,
21734            "workflow_arguments": empty_arguments.clone(),
21735            "query_arguments": empty_arguments,
21736            "history_events": [],
21737            "history_export": {
21738                "payloads": {"codec": DEFAULT_CODEC},
21739                "history_events": [
21740                    {
21741                        "type": "ActivityCompleted",
21742                        "payload": {
21743                            "sequence": 1,
21744                            "activity_type": "load-counter",
21745                            "payload_codec": DEFAULT_CODEC,
21746                            "result": null
21747                        }
21748                    },
21749                    {
21750                        "type": "SignalWaitOpened",
21751                        "payload": {
21752                            "sequence": 3,
21753                            "signal_name": "increment"
21754                        }
21755                    },
21756                    {
21757                        "type": "SignalReceived",
21758                        "payload": {
21759                            "signal_id": "signal-3",
21760                            "signal_name": "increment",
21761                            "workflow_sequence": 2
21762                        }
21763                    },
21764                    {
21765                        "type": "SignalApplied",
21766                        "payload": {
21767                            "sequence": 3,
21768                            "signal_id": "signal-3",
21769                            "signal_name": "increment"
21770                        }
21771                    },
21772                    {
21773                        "type": "SignalWaitOpened",
21774                        "payload": {
21775                            "sequence": 5,
21776                            "signal_name": "increment"
21777                        }
21778                    },
21779                    {
21780                        "type": "SignalReceived",
21781                        "payload": {
21782                            "signal_id": "signal-5",
21783                            "signal_name": "increment",
21784                            "workflow_sequence": 4
21785                        }
21786                    },
21787                    {
21788                        "type": "SignalApplied",
21789                        "payload": {
21790                            "sequence": 5,
21791                            "signal_id": "signal-5",
21792                            "signal_name": "increment"
21793                        }
21794                    }
21795                ],
21796                "activities": [{
21797                    "sequence": 1,
21798                    "activity_type": "load-counter",
21799                    "payload_codec": DEFAULT_CODEC,
21800                    "result": loaded_result
21801                }],
21802                "signals": [
21803                    {
21804                        "id": "signal-3",
21805                        "name": "increment",
21806                        "workflow_sequence": 2,
21807                        "payload_codec": DEFAULT_CODEC,
21808                        "arguments": signal_three
21809                    },
21810                    {
21811                        "id": "signal-5",
21812                        "name": "increment",
21813                        "workflow_sequence": 4,
21814                        "payload_codec": DEFAULT_CODEC,
21815                        "arguments": signal_five
21816                    }
21817                ]
21818            },
21819            "run_status": "completed"
21820        }))
21821        .expect("cold replay query task");
21822        let completed = restarted_worker
21823            .execute_query_task(restarted_task)
21824            .await
21825            .expect("completed cold replay query");
21826        assert_eq!(
21827            completed.into_json().expect("query projection"),
21828            json!({"loaded": "loaded", "count": 8, "finished": true})
21829        );
21830    }
21831
21832    #[tokio::test]
21833    async fn replayed_query_replay_failures_are_machine_readable() {
21834        let worker = replay_counter_worker();
21835        let task = replay_counter_query(
21836            "current",
21837            json!([{
21838                "type": "ActivityCompleted",
21839                "payload": {
21840                    "sequence": 1,
21841                    "payload_codec": DEFAULT_CODEC,
21842                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21843                }
21844            }]),
21845            "running",
21846        );
21847        let failure = worker
21848            .execute_query_task(task)
21849            .await
21850            .expect_err("invalid replay history payload");
21851        assert_eq!(failure.reason, "query_payload_decode_failed");
21852        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21853        assert!(failure.message.contains("invalid_payload_framing"));
21854    }
21855
21856    #[tokio::test]
21857    async fn query_task_restores_compact_history_from_export() {
21858        let client = Client::new("http://127.0.0.1:8080").expect("client");
21859        let mut worker = Worker::new(client, "rust-workers");
21860        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21861        worker.register_query("counter", "current", |ctx, _args| async move {
21862            Ok(json!(ctx.signals("increment")[0][0]))
21863        });
21864        let empty_arguments = fixture_envelope(json!([]));
21865        let exported_signal = fixture_blob(json!([9]));
21866        let task: QueryTask = serde_json::from_value(json!({
21867            "query_task_id": "query-export",
21868            "workflow_type": "counter",
21869            "query_name": "current",
21870            "payload_codec": DEFAULT_CODEC,
21871            "workflow_arguments": empty_arguments.clone(),
21872            "query_arguments": empty_arguments,
21873            "history_events": [],
21874            "history_export": {
21875                "payloads": {"codec": DEFAULT_CODEC},
21876                "history_events": [{
21877                    "type": "SignalReceived",
21878                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21879                }],
21880                "signals": [{
21881                    "id": "signal-export",
21882                    "name": "increment",
21883                    "status": "applied",
21884                    "workflow_sequence": 1,
21885                    "payload_codec": DEFAULT_CODEC,
21886                    "arguments": exported_signal
21887                }]
21888            }
21889        }))
21890        .expect("query task");
21891
21892        let result = worker.execute_query_task(task).await.expect("query result");
21893        assert_eq!(result.into_json().expect("query projection"), json!(9));
21894    }
21895
21896    #[tokio::test]
21897    async fn query_task_failures_have_stable_reasons() {
21898        let client = Client::new("http://127.0.0.1:8080").expect("client");
21899        let mut worker = Worker::new(client, "rust-workers");
21900        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21901        worker.register_query(
21902            "counter",
21903            "current",
21904            |_ctx, _args| async move { Ok(json!(0)) },
21905        );
21906
21907        let base_task = QueryTask {
21908            query_task_id: "query-errors".to_string(),
21909            query_task_attempt: 1,
21910            lease_owner: None,
21911            workflow_id: Some("counter-errors".to_string()),
21912            run_id: Some("run-errors".to_string()),
21913            workflow_type: "counter".to_string(),
21914            query_name: "missing".to_string(),
21915            payload_codec: DEFAULT_CODEC.to_string(),
21916            workflow_arguments: Some(fixture_envelope(json!([]))),
21917            query_arguments: Some(fixture_envelope(json!([]))),
21918            history_events: Vec::new(),
21919            history_export: None,
21920            run_status: Some("running".to_string()),
21921        };
21922
21923        let unknown = worker
21924            .execute_query_task(base_task.clone())
21925            .await
21926            .expect_err("unknown query");
21927        assert_eq!(unknown.reason, "rejected_unknown_query");
21928
21929        let mut malformed = base_task;
21930        malformed.query_name = "current".to_string();
21931        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
21932        let malformed = worker
21933            .execute_query_task(malformed)
21934            .await
21935            .expect_err("malformed payload");
21936        assert_eq!(malformed.reason, "query_payload_decode_failed");
21937
21938        let client = Client::new("http://127.0.0.1:8080").expect("client");
21939        let mut unavailable_worker = Worker::new(client, "rust-workers");
21940        unavailable_worker
21941            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21942        let empty_arguments = fixture_envelope(json!([]));
21943        let unavailable_task: QueryTask = serde_json::from_value(json!({
21944            "query_task_id": "query-unavailable",
21945            "workflow_type": "counter",
21946            "query_name": "current",
21947            "payload_codec": DEFAULT_CODEC,
21948            "workflow_arguments": empty_arguments.clone(),
21949            "query_arguments": empty_arguments
21950        }))
21951        .expect("query task");
21952        let unavailable = unavailable_worker
21953            .execute_query_task(unavailable_task)
21954            .await
21955            .expect_err("query handler unavailable");
21956        assert_eq!(unavailable.reason, "query_handler_unavailable");
21957    }
21958
21959    #[tokio::test]
21960    async fn client_query_decodes_result_and_typed_failure() {
21961        let server = MockWorkerServer::start();
21962        let client = Client::builder(server.base_url())
21963            .timeout(Duration::from_secs(2))
21964            .build()
21965            .expect("client");
21966
21967        let result = client
21968            .query_workflow("counter-1", "current", json!([]))
21969            .await
21970            .expect("query result");
21971        assert_eq!(result, json!({"count": 8}));
21972
21973        let error = client
21974            .query_workflow("counter-1", "missing", json!([]))
21975            .await
21976            .expect_err("unknown query");
21977        let Error::QueryFailed(failure) = error else {
21978            panic!("expected typed query failure");
21979        };
21980        assert_eq!(failure.status, 404);
21981        assert_eq!(failure.reason, "rejected_unknown_query");
21982    }
21983
21984    #[tokio::test]
21985    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
21986        let server = MockWorkerServer::start();
21987        let client = Client::builder(server.base_url())
21988            .timeout(Duration::from_secs(2))
21989            .build()
21990            .expect("client");
21991        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
21992
21993        client
21994            .start_workflow(
21995                "typed.echo",
21996                "rust-workers",
21997                "typed-start",
21998                arguments.clone(),
21999            )
22000            .await
22001            .expect("typed workflow start");
22002        assert_eq!(
22003            decode_wire_avro_value(
22004                &server.request_body("/api/workflows")["input"],
22005                DEFAULT_CODEC,
22006            )
22007            .expect("typed start input"),
22008            arguments
22009        );
22010
22011        client
22012            .signal_workflow("typed-1", "changed", arguments.clone())
22013            .await
22014            .expect("typed signal");
22015        assert_eq!(
22016            decode_wire_avro_value(
22017                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
22018                DEFAULT_CODEC,
22019            )
22020            .expect("typed signal input"),
22021            arguments
22022        );
22023
22024        assert_eq!(
22025            client
22026                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22027                .await
22028                .expect("typed query"),
22029            typed_fidelity_probe()
22030        );
22031        assert_eq!(
22032            decode_wire_avro_value(
22033                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22034                DEFAULT_CODEC,
22035            )
22036            .expect("typed query input"),
22037            arguments
22038        );
22039
22040        assert_eq!(
22041            client
22042                .update_workflow_avro_value(
22043                    "typed-1",
22044                    "replace",
22045                    arguments.clone(),
22046                    Some("typed-request"),
22047                )
22048                .await
22049                .expect("typed update"),
22050            typed_fidelity_probe()
22051        );
22052        let update = server.request_body("/api/workflows/typed-1/update/replace");
22053        assert_eq!(update["request_id"], "typed-request");
22054        assert_eq!(
22055            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22056            arguments
22057        );
22058
22059        let handle = WorkflowHandle {
22060            client: client.clone(),
22061            workflow_id: "typed-1".to_string(),
22062            run_id: Some("run-typed-1".to_string()),
22063            workflow_type: "typed.echo".to_string(),
22064        };
22065        assert_eq!(
22066            handle
22067                .result_avro_value(WorkflowResultOptions::default())
22068                .await
22069                .expect("typed workflow result"),
22070            typed_fidelity_probe()
22071        );
22072
22073        client
22074            .complete_activity_task(
22075                "activity-typed",
22076                "attempt-typed",
22077                "rust-worker",
22078                typed_fidelity_probe(),
22079                DEFAULT_CODEC,
22080            )
22081            .await
22082            .expect("typed activity completion");
22083        assert_eq!(
22084            decode_wire_avro_value(
22085                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22086                    ["result"],
22087                DEFAULT_CODEC,
22088            )
22089            .expect("typed activity result"),
22090            typed_fidelity_probe()
22091        );
22092        client
22093            .fail_activity_task(
22094                "activity-typed",
22095                "attempt-typed",
22096                "rust-worker",
22097                "typed failure",
22098                true,
22099            )
22100            .await
22101            .expect("activity failure");
22102    }
22103
22104    #[tokio::test]
22105    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22106        let server = MockWorkerServer::start();
22107        let client = Client::builder(server.base_url())
22108            .timeout(Duration::from_secs(2))
22109            .build()
22110            .expect("client");
22111
22112        let options = WorkflowCommandOptions::new()
22113            .reason("cleanup requested")
22114            .request_id("cancel-17");
22115        let cancelled = client
22116            .cancel_workflow("wf-lifecycle", options)
22117            .await
22118            .expect("instance cancellation");
22119        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22120        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22121        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22122        assert_eq!(
22123            server.request_body("/api/workflows/wf-lifecycle/cancel"),
22124            json!({"reason":"cleanup requested","request_id":"cancel-17"})
22125        );
22126
22127        let terminated = client
22128            .terminate_workflow(
22129                "wf-lifecycle",
22130                WorkflowCommandOptions::new().reason("forced stop"),
22131            )
22132            .await
22133            .expect("instance termination");
22134        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22135        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22136
22137        client
22138            .cancel_workflow_run(
22139                "wf-lifecycle",
22140                "run-current",
22141                WorkflowCommandOptions::default(),
22142            )
22143            .await
22144            .expect("selected run cancellation");
22145        client
22146            .terminate_workflow_run(
22147                "wf-lifecycle",
22148                "run-current",
22149                WorkflowCommandOptions::default(),
22150            )
22151            .await
22152            .expect("selected run termination");
22153
22154        for (command, error) in [
22155            (
22156                WorkflowCommandKind::Cancel,
22157                client
22158                    .cancel_workflow_run(
22159                        "wf-lifecycle",
22160                        "run-stale",
22161                        WorkflowCommandOptions::default(),
22162                    )
22163                    .await
22164                    .expect_err("stale cancellation must be rejected"),
22165            ),
22166            (
22167                WorkflowCommandKind::Terminate,
22168                client
22169                    .terminate_workflow_run(
22170                        "wf-lifecycle",
22171                        "run-stale",
22172                        WorkflowCommandOptions::default(),
22173                    )
22174                    .await
22175                    .expect_err("stale termination must be rejected"),
22176            ),
22177        ] {
22178            let Error::WorkflowCommandRejected(rejection) = error else {
22179                panic!("expected typed command rejection");
22180            };
22181            assert_eq!(rejection.command, command);
22182            assert_eq!(rejection.status, 409);
22183            assert_eq!(rejection.reason, "historical_run_command_rejected");
22184            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22185            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22186        }
22187    }
22188
22189    #[tokio::test]
22190    async fn workflow_start_options_send_server_enforced_deadlines() {
22191        let server = MockWorkerServer::start();
22192        let client = Client::builder(server.base_url())
22193            .timeout(Duration::from_secs(2))
22194            .build()
22195            .expect("client");
22196
22197        let handle = client
22198            .start_workflow_with_options(
22199                "rust.timeout",
22200                "rust-timeouts",
22201                "wf-start-options",
22202                WorkflowStartOptions::new()
22203                    .execution_timeout_seconds(30)
22204                    .run_timeout_seconds(1),
22205                json!([]),
22206            )
22207            .await
22208            .expect("workflow start");
22209
22210        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22211        let body = server.request_body("/api/workflows");
22212        assert_eq!(body["execution_timeout_seconds"], 30);
22213        assert_eq!(body["run_timeout_seconds"], 1);
22214
22215        let invalid = client
22216            .start_workflow_with_options(
22217                "rust.timeout",
22218                "rust-timeouts",
22219                "wf-invalid-options",
22220                WorkflowStartOptions::new()
22221                    .execution_timeout_seconds(1)
22222                    .run_timeout_seconds(2),
22223                json!([]),
22224            )
22225            .await
22226            .expect_err("invalid deadline ordering");
22227        assert!(invalid
22228            .to_string()
22229            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22230    }
22231
22232    #[tokio::test]
22233    async fn workflow_result_returns_each_typed_terminal_outcome() {
22234        let server = MockWorkerServer::start();
22235        let client = Client::builder(server.base_url())
22236            .timeout(Duration::from_secs(2))
22237            .build()
22238            .expect("client");
22239        let options = WorkflowResultOptions {
22240            poll_interval: Duration::ZERO,
22241            timeout: Duration::from_secs(1),
22242        };
22243
22244        let failed = WorkflowHandle {
22245            client: client.clone(),
22246            workflow_id: "wf-failed".to_string(),
22247            run_id: Some("run-failed".to_string()),
22248            workflow_type: "failure".to_string(),
22249        }
22250        .result(options)
22251        .await
22252        .expect_err("failed outcome");
22253        let Error::WorkflowFailed(failure) = failed else {
22254            panic!("expected WorkflowFailed");
22255        };
22256        assert_eq!(failure.workflow_id, "wf-failed");
22257        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22258        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22259        assert_eq!(failure.failure_category.as_deref(), Some("application"));
22260        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22261        assert_eq!(
22262            failure.exception_class.as_deref(),
22263            Some("billing::PaymentError")
22264        );
22265        assert_eq!(failure.non_retryable, Some(true));
22266
22267        for (workflow_id, expected_kind, expected_reason) in [
22268            (
22269                "wf-cancelled",
22270                WorkflowTerminalKind::Cancelled,
22271                "cleanup requested",
22272            ),
22273            (
22274                "wf-terminated",
22275                WorkflowTerminalKind::Terminated,
22276                "forced stop",
22277            ),
22278            (
22279                "wf-timed-out",
22280                WorkflowTerminalKind::TimedOut,
22281                "run_timeout",
22282            ),
22283        ] {
22284            let error = WorkflowHandle {
22285                client: client.clone(),
22286                workflow_id: workflow_id.to_string(),
22287                run_id: None,
22288                workflow_type: "terminal".to_string(),
22289            }
22290            .result(options)
22291            .await
22292            .expect_err("typed terminal outcome");
22293            let outcome = match error {
22294                Error::WorkflowCancelled(outcome) => outcome,
22295                Error::WorkflowTerminated(outcome) => outcome,
22296                Error::WorkflowTimedOut(outcome) => outcome,
22297                other => panic!("unexpected terminal error: {other}"),
22298            };
22299            assert_eq!(outcome.kind, expected_kind);
22300            assert_eq!(outcome.workflow_id, workflow_id);
22301            assert_eq!(outcome.reason, expected_reason);
22302        }
22303
22304        let wait_timeout = WorkflowHandle {
22305            client,
22306            workflow_id: "wf-waiting".to_string(),
22307            run_id: Some("run-waiting".to_string()),
22308            workflow_type: "waiting".to_string(),
22309        }
22310        .result(WorkflowResultOptions {
22311            poll_interval: Duration::ZERO,
22312            timeout: Duration::ZERO,
22313        })
22314        .await
22315        .expect_err("client wait timeout");
22316        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22317            panic!("expected typed client timeout");
22318        };
22319        assert_eq!(timeout.reason, "result_wait_timeout");
22320        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22321        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22322    }
22323
22324    #[tokio::test]
22325    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22326        let server = MockWorkerServer::start();
22327        let client = Client::builder(server.base_url())
22328            .timeout(Duration::from_secs(2))
22329            .build()
22330            .expect("client");
22331
22332        let handle = WorkflowHandle {
22333            client,
22334            workflow_id: "wf-selected".to_string(),
22335            run_id: Some("run-selected".to_string()),
22336            workflow_type: "selected".to_string(),
22337        };
22338        let options = WorkflowResultOptions {
22339            poll_interval: Duration::ZERO,
22340            timeout: Duration::from_secs(1),
22341        };
22342
22343        let current = handle
22344            .result(options)
22345            .await
22346            .expect("instance result follows the current run");
22347        assert_eq!(current, json!("current run output"));
22348
22349        let error = handle
22350            .result_selected_run(options)
22351            .await
22352            .expect_err("the selected run is cancelled even though the current run completed");
22353
22354        let Error::WorkflowCancelled(outcome) = error else {
22355            panic!("expected selected run cancellation");
22356        };
22357        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22358        assert_eq!(outcome.reason, "selected run cancelled");
22359        assert_eq!(
22360            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22361            1
22362        );
22363        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22364    }
22365
22366    #[tokio::test]
22367    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22368        let server = MockWorkerServer::draining_polls();
22369        let client = Client::builder(server.base_url())
22370            .timeout(Duration::from_secs(2))
22371            .build()
22372            .expect("client");
22373
22374        let workflow = client
22375            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22376            .await
22377            .expect("workflow drain response");
22378        let activity = client
22379            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22380            .await
22381            .expect("activity drain response");
22382        let query = client
22383            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22384            .await
22385            .expect("query drain response");
22386
22387        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22388            assert_eq!(
22389                outcome,
22390                WorkerPollOutcome::Stop {
22391                    poll_status: Some("draining".to_string()),
22392                    reason: Some("worker_draining".to_string()),
22393                }
22394            );
22395        }
22396
22397        assert!(client
22398            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22399            .await
22400            .expect("compatibility poll")
22401            .is_none());
22402    }
22403
22404    #[tokio::test]
22405    async fn managed_worker_honors_drain_stop_for_every_task_family() {
22406        let server = MockWorkerServer::draining_polls();
22407        let client = Client::builder(server.base_url())
22408            .timeout(Duration::from_secs(2))
22409            .build()
22410            .expect("client");
22411
22412        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22413            .worker_id("draining-workflow-worker")
22414            .poll_timeout(Duration::ZERO);
22415        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22416        workflow_worker
22417            .run()
22418            .await
22419            .expect("workflow drain is a clean stop");
22420
22421        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22422            .worker_id("draining-activity-worker")
22423            .poll_timeout(Duration::ZERO);
22424        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22425        activity_worker
22426            .run()
22427            .await
22428            .expect("activity drain is a clean stop");
22429
22430        let mut query_worker = Worker::new(client, "rust-workers")
22431            .worker_id("draining-query-worker")
22432            .poll_timeout(Duration::ZERO);
22433        query_worker.register_query("counter", "current", |_ctx, _args| async {
22434            Ok(Value::Null)
22435        });
22436        query_worker
22437            .run()
22438            .await
22439            .expect("query drain is a clean stop");
22440    }
22441
22442    #[tokio::test]
22443    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22444        let server = MockWorkerServer::start();
22445        let client = Client::builder(server.base_url())
22446            .timeout(Duration::from_secs(2))
22447            .build()
22448            .expect("client");
22449
22450        let heartbeat = client
22451            .heartbeat_activity_task(
22452                "activity-cancel",
22453                "attempt-cancel",
22454                "rust-worker",
22455                typed_fidelity_probe(),
22456            )
22457            .await
22458            .expect("cancellation heartbeat");
22459        assert!(heartbeat.cancel_requested);
22460        assert!(heartbeat.should_stop());
22461        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22462        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22463        let heartbeat_body =
22464            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22465        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22466        assert_eq!(
22467            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22468                .expect("typed heartbeat details"),
22469            typed_fidelity_probe()
22470        );
22471
22472        let error = client
22473            .complete_activity_task(
22474                "activity-cancel",
22475                "attempt-cancel",
22476                "rust-worker",
22477                json!({"late":true}),
22478                DEFAULT_CODEC,
22479            )
22480            .await
22481            .expect_err("late completion must be refused");
22482        assert!(activity_task_rejection_is_final(&error));
22483        let Error::ActivityTaskRejected(rejection) = error else {
22484            panic!("expected typed activity rejection");
22485        };
22486        assert_eq!(rejection.status, 409);
22487        assert_eq!(rejection.reason, "run_cancelled");
22488        assert!(rejection.cancel_requested);
22489        assert_eq!(rejection.can_continue, Some(false));
22490    }
22491
22492    #[tokio::test]
22493    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22494        let server = MockWorkerServer::cancelled_activity();
22495        let client = Client::builder(server.base_url())
22496            .timeout(Duration::from_secs(2))
22497            .build()
22498            .expect("client");
22499        let cancellation_observed = Arc::new(AtomicBool::new(false));
22500        let observed = Arc::clone(&cancellation_observed);
22501        let mut worker = Worker::new(client.clone(), "rust-workers")
22502            .worker_id("rust-cancel-worker")
22503            .poll_timeout(Duration::from_millis(10));
22504        worker.register_activity("cancel-aware", move |ctx, _args| {
22505            let observed = Arc::clone(&observed);
22506            async move {
22507                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22508                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22509                Ok(json!({"late":"completion"}))
22510            }
22511        });
22512
22513        assert_eq!(
22514            worker.run_once().await.expect("cancelled attempt handled"),
22515            1
22516        );
22517        assert!(cancellation_observed.load(Ordering::SeqCst));
22518        assert_eq!(
22519            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22520            1
22521        );
22522
22523        let mut restarted = Worker::new(client, "rust-workers")
22524            .worker_id("rust-cancel-worker-restarted")
22525            .poll_timeout(Duration::from_millis(10));
22526        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22527        assert_eq!(
22528            restarted
22529                .run_once()
22530                .await
22531                .expect("replacement worker continues polling"),
22532            0
22533        );
22534    }
22535
22536    #[tokio::test]
22537    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22538        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22539        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22540        let client = Client::builder(server.base_url())
22541            .timeout(Duration::from_secs(2))
22542            .build()
22543            .expect("client");
22544
22545        let direct_error = client
22546            .complete_workflow_task(
22547                "workflow-timeout-task",
22548                "timeout-worker",
22549                3,
22550                vec![json!({
22551                    "type": "complete_workflow",
22552                    "result": fixture_envelope(Value::Null)
22553                })],
22554            )
22555            .await
22556            .expect_err("the low-level client preserves the completion rejection");
22557        let Error::Http { status, body } = direct_error else {
22558            panic!("expected the original HTTP completion rejection");
22559        };
22560        assert_eq!(status, reqwest::StatusCode::CONFLICT);
22561        assert_eq!(
22562            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22563            "run_timed_out"
22564        );
22565
22566        let mut worker = Worker::new(client, "rust-workers")
22567            .worker_id("timeout-worker")
22568            .poll_timeout(Duration::from_millis(10));
22569        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22570            Ok(json!({"late": "result"}))
22571        });
22572
22573        assert_eq!(
22574            worker
22575                .run_once()
22576                .await
22577                .expect("authoritative selected-run timeout settles the tick"),
22578            1
22579        );
22580        assert_eq!(
22581            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22582            2,
22583            "both the direct client proof and managed worker must see the rejection"
22584        );
22585    }
22586
22587    #[tokio::test]
22588    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22589        for (name, status, response) in [
22590            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22591            (
22592                "command was recorded",
22593                "409 Conflict",
22594                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22595            ),
22596            (
22597                "lease conflict",
22598                "409 Conflict",
22599                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22600            ),
22601            (
22602                "nonterminal run",
22603                "409 Conflict",
22604                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22605            ),
22606            (
22607                "different selected run",
22608                "409 Conflict",
22609                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22610            ),
22611            (
22612                "different task attempt",
22613                "409 Conflict",
22614                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22615            ),
22616            (
22617                "authentication failure",
22618                "401 Unauthorized",
22619                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22620            ),
22621            (
22622                "authorization failure",
22623                "403 Forbidden",
22624                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22625            ),
22626            (
22627                "protocol failure",
22628                "400 Bad Request",
22629                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22630            ),
22631            (
22632                "malformed command",
22633                "422 Unprocessable Entity",
22634                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22635            ),
22636            (
22637                "transient server failure",
22638                "503 Service Unavailable",
22639                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22640            ),
22641        ] {
22642            let server = MockWorkerServer::workflow_completion(status, response);
22643            let client = Client::builder(server.base_url())
22644                .timeout(Duration::from_secs(2))
22645                .build()
22646                .expect("client");
22647            let mut worker = Worker::new(client, "rust-workers")
22648                .worker_id("timeout-worker")
22649                .poll_timeout(Duration::from_millis(10));
22650            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22651                Ok(json!({"late": "result"}))
22652            });
22653
22654            let error = worker
22655                .run_once()
22656                .await
22657                .expect_err(&format!("{name} must remain an error"));
22658            assert!(
22659                matches!(error, Error::Http { .. } | Error::Protocol(_)),
22660                "{name} returned an unexpected error variant: {error}"
22661            );
22662        }
22663    }
22664
22665    #[tokio::test]
22666    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22667        let server = MockWorkerServer::start();
22668        let client = Client::builder(server.base_url())
22669            .worker_token(Some("worker-secret".to_string()))
22670            .namespace("orders")
22671            .timeout(Duration::from_secs(2))
22672            .build()
22673            .expect("client");
22674        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22675
22676        let result = client
22677            .deregister_worker_registration("worker/α space")
22678            .await
22679            .expect("deregister worker registration");
22680
22681        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22682        assert_eq!(
22683            server.worker_protocol_for(path).as_deref(),
22684            Some(WORKER_PROTOCOL_VERSION)
22685        );
22686        assert_eq!(server.control_protocol_for(path), None);
22687        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22688        assert_eq!(
22689            server.authorization_for(path).as_deref(),
22690            Some("Bearer worker-secret")
22691        );
22692        assert_eq!(
22693            result,
22694            WorkerDeregistrationEnvelope {
22695                worker_id: "deregistered-worker".to_string(),
22696                outcome: "deregistered".to_string(),
22697                recovered_workflow_task_count: 2,
22698            }
22699        );
22700    }
22701
22702    #[tokio::test]
22703    async fn low_level_registration_rejects_update_validators_before_transport() {
22704        let server = MockWorkerServer::start();
22705        let client = Client::builder(server.base_url())
22706            .timeout(Duration::from_secs(2))
22707            .build()
22708            .expect("client");
22709
22710        for update_validators in [json!(["approve"]), json!("approve")] {
22711            let error = client
22712                .register_worker_with_command_contracts(
22713                    "validator-claiming-worker",
22714                    "rust-workers",
22715                    vec!["orders".to_string()],
22716                    vec![],
22717                    1,
22718                    1,
22719                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22720                    json!({
22721                        "orders": {
22722                            "queries": ["current"],
22723                            "updates": ["approve"],
22724                            "update_validators": update_validators,
22725                        },
22726                    }),
22727                )
22728                .await
22729                .expect_err("unsupported validator claims must fail before registration");
22730
22731            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22732                panic!("expected typed unsupported-validator failure");
22733            };
22734            assert_eq!(workflow_type, "orders");
22735        }
22736        assert_eq!(server.request_count("/api/worker/register"), 0);
22737    }
22738
22739    #[tokio::test]
22740    async fn low_level_registration_preserves_query_and_update_contracts() {
22741        let server = MockWorkerServer::start();
22742        let client = Client::builder(server.base_url())
22743            .timeout(Duration::from_secs(2))
22744            .build()
22745            .expect("client");
22746        let contracts = json!({
22747            "orders": {
22748                "queries": ["current"],
22749                "updates": ["approve"],
22750                "update_validators": [],
22751            },
22752            "payments": {
22753                "queries": ["status"],
22754                "updates": ["capture"],
22755            },
22756        });
22757
22758        client
22759            .register_worker_with_command_contracts(
22760                "command-worker",
22761                "rust-workers",
22762                vec!["orders".to_string(), "payments".to_string()],
22763                vec![],
22764                1,
22765                1,
22766                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22767                contracts.clone(),
22768            )
22769            .await
22770            .expect("query and update contracts must remain supported");
22771
22772        assert_eq!(
22773            server.request_body("/api/worker/register")["workflow_command_contracts"],
22774            contracts
22775        );
22776    }
22777
22778    #[tokio::test]
22779    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22780        let server = MockWorkerServer::start();
22781        let control_only = Client::builder(server.base_url())
22782            .control_token(Some("control-secret".to_string()))
22783            .build()
22784            .expect("control client");
22785
22786        let error = control_only
22787            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22788            .await
22789            .expect_err("control token must not authorize a worker request");
22790        assert!(matches!(
22791            error,
22792            Error::MissingRoleCredentials { role: "worker", .. }
22793        ));
22794        assert_eq!(server.request_count("/api/worker/register"), 0);
22795
22796        let worker_only = Client::builder(server.base_url())
22797            .worker_token(Some("worker-secret".to_string()))
22798            .build()
22799            .expect("worker client");
22800        let error = worker_only
22801            .health()
22802            .await
22803            .expect_err("worker token must not authorize a control request");
22804        assert!(matches!(
22805            error,
22806            Error::MissingRoleCredentials {
22807                role: "control",
22808                ..
22809            }
22810        ));
22811        assert_eq!(server.request_count("/api/health"), 0);
22812    }
22813
22814    #[tokio::test]
22815    async fn shared_token_supports_worker_and_control_planes() {
22816        let server = MockWorkerServer::start();
22817        let client = Client::builder(server.base_url())
22818            .token(Some("shared-secret".to_string()))
22819            .build()
22820            .expect("client");
22821
22822        client.health().await.expect("control request");
22823        client
22824            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22825            .await
22826            .expect("worker request");
22827
22828        assert_eq!(
22829            server.authorization_for("/api/health").as_deref(),
22830            Some("Bearer shared-secret")
22831        );
22832        assert_eq!(
22833            server.control_protocol_for("/api/health").as_deref(),
22834            Some(CONTROL_PLANE_VERSION)
22835        );
22836        assert_eq!(
22837            server.authorization_for("/api/worker/register").as_deref(),
22838            Some("Bearer shared-secret")
22839        );
22840        assert_eq!(
22841            server
22842                .worker_protocol_for("/api/worker/register")
22843                .as_deref(),
22844            Some(WORKER_PROTOCOL_VERSION)
22845        );
22846    }
22847
22848    #[tokio::test]
22849    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22850        let server = MockWorkerServer::start();
22851        let client = Client::builder(server.base_url())
22852            .timeout(Duration::from_secs(2))
22853            .build()
22854            .expect("client");
22855
22856        client
22857            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22858            .await
22859            .expect("register");
22860        client
22861            .heartbeat_worker("capture-worker", 1, 1)
22862            .await
22863            .expect("heartbeat");
22864        client
22865            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22866            .await
22867            .expect("workflow poll");
22868        client
22869            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22870            .await
22871            .expect("activity poll");
22872
22873        for path in [
22874            "/api/worker/register",
22875            "/api/worker/heartbeat",
22876            "/api/worker/workflow-tasks/poll",
22877            "/api/worker/activity-tasks/poll",
22878        ] {
22879            assert_eq!(
22880                server.worker_protocol_for(path).as_deref(),
22881                Some(WORKER_PROTOCOL_VERSION),
22882                "unexpected protocol for {path}"
22883            );
22884        }
22885
22886        assert_eq!(
22887            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22888            1
22889        );
22890        assert_eq!(
22891            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22892            1
22893        );
22894        assert!(
22895            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22896                .as_str()
22897                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22898        );
22899        assert!(
22900            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22901                .as_str()
22902                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22903        );
22904    }
22905
22906    #[tokio::test]
22907    async fn query_task_endpoints_send_the_query_feature_protocol() {
22908        let server = MockWorkerServer::start();
22909        let client = Client::builder(server.base_url())
22910            .timeout(Duration::from_secs(2))
22911            .build()
22912            .expect("client");
22913
22914        client
22915            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22916            .await
22917            .expect("query poll");
22918        client
22919            .complete_query_task(
22920                "query-capture",
22921                "capture-worker",
22922                1,
22923                json!(8),
22924                DEFAULT_CODEC,
22925            )
22926            .await
22927            .expect("query complete");
22928        client
22929            .fail_query_task(
22930                "query-capture",
22931                "capture-worker",
22932                1,
22933                "failed",
22934                "query_rejected",
22935                "QueryFailed",
22936            )
22937            .await
22938            .expect("query fail");
22939
22940        for path in [
22941            "/api/worker/query-tasks/poll",
22942            "/api/worker/query-tasks/query-capture/complete",
22943            "/api/worker/query-tasks/query-capture/fail",
22944        ] {
22945            assert_eq!(
22946                server.worker_protocol_for(path).as_deref(),
22947                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
22948                "unexpected protocol for {path}"
22949            );
22950        }
22951
22952        assert_eq!(
22953            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
22954            1
22955        );
22956        assert!(
22957            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
22958                .as_str()
22959                .is_some_and(|id| id.starts_with("rust-query-poll-"))
22960        );
22961    }
22962
22963    #[tokio::test]
22964    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
22965        let server = MockWorkerServer::transient_worker_failures();
22966        let client = Client::builder(server.base_url())
22967            .timeout(Duration::from_secs(2))
22968            .build()
22969            .expect("client");
22970
22971        client
22972            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22973            .await
22974            .expect("workflow poll retry");
22975        client
22976            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22977            .await
22978            .expect("activity poll retry");
22979        client
22980            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22981            .await
22982            .expect("query poll retry");
22983
22984        for path in [
22985            "/api/worker/workflow-tasks/poll",
22986            "/api/worker/activity-tasks/poll",
22987            "/api/worker/query-tasks/poll",
22988        ] {
22989            let bodies = server.request_bodies(path);
22990            assert_eq!(bodies.len(), 2, "{path} must be retried once");
22991            assert_eq!(
22992                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
22993                "{path} must preserve the request binding across retry"
22994            );
22995        }
22996    }
22997
22998    #[tokio::test]
22999    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
23000        let server = MockWorkerServer::consecutive_poll_failures(2);
23001        let client = Client::builder(server.base_url())
23002            .timeout(Duration::from_secs(2))
23003            .build()
23004            .expect("client");
23005        let mut worker = Worker::new(client, "capture")
23006            .worker_id("capture-worker")
23007            .poll_timeout(Duration::from_millis(10))
23008            .retry_policy(WorkerRetryPolicy {
23009                max_retries: 2,
23010                initial_backoff: Duration::from_millis(1),
23011                max_backoff: Duration::from_millis(1),
23012            });
23013        worker.register_workflow(
23014            "capture.workflow",
23015            |_ctx, _input| async move { Ok(Value::Null) },
23016        );
23017        worker.register_activity(
23018            "capture.activity",
23019            |_ctx, _input| async move { Ok(Value::Null) },
23020        );
23021        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23022            Ok(Value::Null)
23023        });
23024
23025        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23026
23027        for path in [
23028            "/api/worker/workflow-tasks/poll",
23029            "/api/worker/activity-tasks/poll",
23030            "/api/worker/query-tasks/poll",
23031        ] {
23032            let bodies = server.request_bodies(path);
23033            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23034            assert!(
23035                bodies
23036                    .iter()
23037                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23038                "{path} must preserve one request binding across every retry"
23039            );
23040        }
23041    }
23042
23043    fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23044        let reason = if unavailable {
23045            "storage_admission_unavailable"
23046        } else {
23047            "storage_pressure"
23048        };
23049        let mut body = json!({
23050            "reason": reason,
23051            "storage_state": if unavailable { "fenced" } else { "draining" },
23052            "retryable": true,
23053            "retry_after_seconds": 1,
23054        });
23055        if !mid_poll {
23056            body["request_admitted"] = json!(false);
23057        }
23058        if let Some(id) = poll_id {
23059            body["task"] = Value::Null;
23060            body["poll_status"] = json!(reason);
23061            body["poll_request_id"] = json!(id);
23062            body["retry_same_poll_request_id"] = json!(true);
23063            body["claim_admitted"] = json!(false);
23064        }
23065        body
23066    }
23067
23068    fn storage_worker(server: &MockWorkerServer) -> Worker {
23069        Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23070            .worker_id("storage-worker")
23071            .retry_policy(WorkerRetryPolicy {
23072                max_retries: 1,
23073                initial_backoff: Duration::from_millis(1),
23074                max_backoff: Duration::from_millis(1),
23075            })
23076    }
23077
23078    fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23079        let requests = server.requests.lock().expect("requests");
23080        let bodies: Vec<_> = requests
23081            .iter()
23082            .filter(|request| request.path == path)
23083            .map(|request| &request.body)
23084            .collect();
23085        assert_eq!(bodies.len(), count, "{path}");
23086        assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23087    }
23088
23089    #[test]
23090    fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23091        for unavailable in [false, true] {
23092            for mid_poll in [false, true] {
23093                let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23094                let error = Error::Http {
23095                    status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23096                    body: body.to_string(),
23097                };
23098                assert_eq!(
23099                    worker_storage_admission_retry_after(&error, Some("same-poll")),
23100                    Some(Duration::from_secs(1))
23101                );
23102                assert!(
23103                    !worker_operation_is_retryable(&error),
23104                    "storage is not a bounded generic retry"
23105                );
23106                for (field, value) in [
23107                    ("poll_request_id", json!("wrong-poll")),
23108                    ("task", json!({"task_id":"claimed"})),
23109                    ("retryable", json!(false)),
23110                    ("retry_after_seconds", json!(0)),
23111                    ("retry_after_seconds", json!(true)),
23112                    ("retry_after_seconds", json!(1.0)),
23113                    ("storage_state", json!("normal")),
23114                    ("poll_status", json!("empty")),
23115                    ("claim_admitted", json!(true)),
23116                    ("retry_same_poll_request_id", json!(false)),
23117                    ("request_admitted", json!(true)),
23118                ] {
23119                    let mut invalid = body.clone();
23120                    invalid[field] = value;
23121                    let error = Error::Http {
23122                        status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23123                        body: invalid.to_string(),
23124                    };
23125                    assert!(
23126                        worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23127                        "{field}"
23128                    );
23129                }
23130            }
23131        }
23132        let body = storage_refusal(None, false, false);
23133        let error = Error::Http {
23134            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23135            body: body.to_string(),
23136        };
23137        assert!(worker_storage_admission_retry_after(&error, None).is_some());
23138        assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23139        let error = Error::Http {
23140            status: reqwest::StatusCode::FORBIDDEN,
23141            body: body.to_string(),
23142        };
23143        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23144        let body = storage_refusal(None, false, true);
23145        let error = Error::Http {
23146            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23147            body: body.to_string(),
23148        };
23149        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23150    }
23151
23152    #[tokio::test]
23153    async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23154        for unavailable in [false, true] {
23155            for mid_poll in [false, true] {
23156                let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23157                    poll_failures_per_path: 1,
23158                    storage_refusals: 7,
23159                    storage_path: Some("/poll"),
23160                    storage_unavailable: unavailable,
23161                    storage_mid_poll: mid_poll,
23162                    ..MockWorkerBehavior::default()
23163                });
23164                let mut worker = storage_worker(&server);
23165                worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23166                assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23167                for path in [
23168                    "/api/worker/workflow-tasks/poll",
23169                    "/api/worker/activity-tasks/poll",
23170                    "/api/worker/query-tasks/poll",
23171                ] {
23172                    assert_identical_requests(&server, path, 9);
23173                }
23174            }
23175        }
23176    }
23177
23178    #[tokio::test]
23179    async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23180        struct CountedBody(Arc<AtomicUsize>);
23181        impl Serialize for CountedBody {
23182            fn serialize<S: Serializer>(
23183                &self,
23184                serializer: S,
23185            ) -> std::result::Result<S::Ok, S::Error> {
23186                let count = self.0.fetch_add(1, Ordering::SeqCst);
23187                json!({"serialization":count,"lease_owner":"worker","attempt":7})
23188                    .serialize(serializer)
23189            }
23190        }
23191        for path in [
23192            "/api/worker/register",
23193            "/api/worker/heartbeat",
23194            "/api/worker/workflow-tasks/storage-task/complete",
23195            "/api/worker/workflow-tasks/storage-task/fail",
23196            "/api/worker/activity-tasks/storage-task/complete",
23197            "/api/worker/activity-tasks/storage-task/fail",
23198            "/api/worker/activity-tasks/storage-task/heartbeat",
23199            "/api/worker/query-tasks/storage-task/complete",
23200            "/api/worker/query-tasks/storage-task/fail",
23201        ] {
23202            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23203                storage_refusals: 7,
23204                storage_path: Some(path),
23205                ..MockWorkerBehavior::default()
23206            });
23207            let worker =
23208                storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23209            let calls = Arc::new(AtomicUsize::new(0));
23210            let _: Value = worker
23211                .client
23212                .request_json(
23213                    reqwest::Method::POST,
23214                    &path[4..],
23215                    RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23216                    Some(&CountedBody(Arc::clone(&calls))),
23217                )
23218                .await
23219                .expect("prepared request recovery");
23220            assert_eq!(calls.load(Ordering::SeqCst), 1);
23221            assert_identical_requests(&server, path, 8);
23222        }
23223        for worker_scope in [false, true] {
23224            let path = if worker_scope {
23225                "/api/health"
23226            } else {
23227                "/api/worker/register"
23228            };
23229            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23230                storage_refusals: usize::MAX,
23231                storage_path: Some(path),
23232                ..MockWorkerBehavior::default()
23233            });
23234            let worker = storage_worker(&server);
23235            let client = worker.client.clone();
23236            let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
23237            let error = if worker_scope {
23238                worker
23239                    .client
23240                    .health()
23241                    .await
23242                    .expect_err("control plane is not retried")
23243            } else {
23244                client
23245                    .request_json::<Value, Value>(
23246                        reqwest::Method::POST,
23247                        "/worker/register",
23248                        RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23249                        Some(&json!({})),
23250                    )
23251                    .await
23252                    .expect_err("direct client is not retried")
23253            };
23254            assert!(worker_storage_admission_body(&error).is_some());
23255            assert_eq!(server.request_count(path), 1);
23256        }
23257    }
23258
23259    #[tokio::test]
23260    async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
23261        for fail in [false, true] {
23262            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23263                storage_activity: true,
23264                storage_refusals: 7,
23265                storage_path: Some("/storage-activity/"),
23266                ..MockWorkerBehavior::default()
23267            });
23268            let mut worker = storage_worker(&server);
23269            let calls = Arc::new(AtomicUsize::new(0));
23270            let observed = Arc::clone(&calls);
23271            worker.register_activity("storage.activity", move |ctx, _| {
23272                let calls = Arc::clone(&observed);
23273                async move {
23274                    calls.fetch_add(1, Ordering::SeqCst);
23275                    ctx.heartbeat(json!({"step":2})).await?;
23276                    if fail {
23277                        Err(Error::WorkerLoop("intentional handler failure".to_string()))
23278                    } else {
23279                        Ok(json!({"receipt":true}))
23280                    }
23281                }
23282            });
23283            assert_eq!(worker.run_once().await.expect("activity settled"), 1);
23284            assert_eq!(calls.load(Ordering::SeqCst), 1);
23285            assert_identical_requests(
23286                &server,
23287                "/api/worker/activity-tasks/storage-activity/heartbeat",
23288                8,
23289            );
23290            let suffix = if fail { "fail" } else { "complete" };
23291            assert_identical_requests(
23292                &server,
23293                &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
23294                8,
23295            );
23296            let other = if fail { "complete" } else { "fail" };
23297            assert_eq!(
23298                server.request_count(&format!(
23299                    "/api/worker/activity-tasks/storage-activity/{other}"
23300                )),
23301                0
23302            );
23303        }
23304    }
23305
23306    #[tokio::test]
23307    async fn storage_waits_are_interruptible_without_false_activity_failure() {
23308        for path in [
23309            "/api/worker/register",
23310            "/api/worker/heartbeat",
23311            "/api/worker/activity-tasks/poll",
23312            "/api/worker/activity-tasks/storage-activity/heartbeat",
23313            "/api/worker/activity-tasks/storage-activity/complete",
23314        ] {
23315            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23316                storage_activity: true,
23317                storage_refusals: usize::MAX,
23318                storage_path: Some(path),
23319                ..MockWorkerBehavior::default()
23320            });
23321            let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23322            worker.register_activity("storage.activity", |ctx, _| async move {
23323                ctx.heartbeat(json!({"step":2})).await?;
23324                Ok(json!({"receipt":true}))
23325            });
23326            let shutdown = async {
23327                while server.request_count(path) == 0 {
23328                    tokio::time::sleep(Duration::from_millis(1)).await;
23329                }
23330            };
23331            let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
23332                .await
23333                .expect("shutdown interrupts admission");
23334            assert!(
23335                result.is_err(),
23336                "a refused operation must not appear acknowledged: {path}, {result:?}"
23337            );
23338            assert_eq!(server.request_count(path), 1);
23339            assert_eq!(
23340                server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
23341                0
23342            );
23343            assert_eq!(
23344                server.request_count("/api/worker/registrations/mock-worker"),
23345                usize::from(!path.ends_with("/register"))
23346            );
23347        }
23348    }
23349
23350    #[tokio::test]
23351    async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
23352        for fail in [false, true] {
23353            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23354                storage_query: true,
23355                storage_refusals: 7,
23356                storage_path: Some("/storage-query/"),
23357                ..MockWorkerBehavior::default()
23358            });
23359            let mut worker = storage_worker(&server);
23360            worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
23361            let calls = Arc::new(AtomicUsize::new(0));
23362            let observed = Arc::clone(&calls);
23363            worker.register_query("storage.workflow", "state", move |_, _| {
23364                let calls = Arc::clone(&observed);
23365                async move {
23366                    calls.fetch_add(1, Ordering::SeqCst);
23367                    if fail {
23368                        Err(Error::WorkerLoop("intentional query failure".to_string()))
23369                    } else {
23370                        Ok(json!({"state":"waiting"}))
23371                    }
23372                }
23373            });
23374            assert_eq!(worker.run_once().await.expect("query settled"), 1);
23375            assert_eq!(calls.load(Ordering::SeqCst), 1);
23376            let suffix = if fail { "fail" } else { "complete" };
23377            assert_identical_requests(
23378                &server,
23379                &format!("/api/worker/query-tasks/storage-query/{suffix}"),
23380                8,
23381            );
23382            let other = if fail { "complete" } else { "fail" };
23383            assert_eq!(
23384                server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
23385                0
23386            );
23387        }
23388    }
23389
23390    #[tokio::test]
23391    async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
23392        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23393            storage_refusals: 7,
23394            storage_path: Some("/poll"),
23395            unauthorized_polls: true,
23396            ..MockWorkerBehavior::default()
23397        });
23398        let error = storage_worker(&server)
23399            .run_once()
23400            .await
23401            .expect_err("auth remains terminal");
23402        assert!(matches!(
23403            error,
23404            Error::Http {
23405                status: reqwest::StatusCode::UNAUTHORIZED,
23406                ..
23407            }
23408        ));
23409        assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
23410
23411        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23412            storage_refusals: 7,
23413            storage_path: Some("/activity-cancel/complete"),
23414            ..MockWorkerBehavior::default()
23415        });
23416        let worker =
23417            storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23418        let error = worker
23419            .client
23420            .complete_activity_task(
23421                "activity-cancel",
23422                "attempt-cancel",
23423                "worker",
23424                json!({}),
23425                DEFAULT_CODEC,
23426            )
23427            .await
23428            .expect_err("cancellation remains terminal");
23429        assert!(activity_task_rejection_is_final(&error));
23430        assert_identical_requests(
23431            &server,
23432            "/api/worker/activity-tasks/activity-cancel/complete",
23433            8,
23434        );
23435
23436        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23437            storage_refusals: usize::MAX,
23438            storage_path: Some("/poll"),
23439            storage_wrong_poll_id: true,
23440            ..MockWorkerBehavior::default()
23441        });
23442        assert!(storage_worker(&server).run_once().await.is_err());
23443        assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
23444    }
23445
23446    #[tokio::test]
23447    async fn storage_pollers_stop_when_the_run_future_is_aborted() {
23448        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23449            storage_refusals: usize::MAX,
23450            storage_path: Some("/poll"),
23451            ..MockWorkerBehavior::default()
23452        });
23453        let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23454        worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
23455        let run = tokio::spawn(async move { worker.run().await });
23456        tokio::time::timeout(Duration::from_secs(2), async {
23457            while server.request_count("/api/worker/activity-tasks/poll") == 0 {
23458                tokio::time::sleep(Duration::from_millis(1)).await;
23459            }
23460        })
23461        .await
23462        .expect("poll started");
23463        run.abort();
23464        assert!(run.await.expect_err("cancelled run").is_cancelled());
23465        tokio::time::sleep(Duration::from_millis(250)).await;
23466        assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
23467    }
23468
23469    #[tokio::test]
23470    async fn query_protocol_rejection_from_older_server_is_typed() {
23471        let server = MockWorkerServer::reject_query_protocol();
23472        let client = Client::builder(server.base_url())
23473            .timeout(Duration::from_secs(2))
23474            .build()
23475            .expect("client");
23476
23477        let error = client
23478            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23479            .await
23480            .expect_err("server below query protocol floor must reject");
23481        let Error::Protocol(failure) = error else {
23482            panic!("expected typed protocol failure");
23483        };
23484
23485        assert_eq!(failure.status, 400);
23486        assert_eq!(failure.reason, "unsupported_protocol_version");
23487        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
23488        assert_eq!(
23489            failure.requested_version.as_deref(),
23490            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23491        );
23492        assert_eq!(
23493            server
23494                .worker_protocol_for("/api/worker/query-tasks/poll")
23495                .as_deref(),
23496            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23497        );
23498    }
23499
23500    #[tokio::test]
23501    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
23502        let server = MockWorkerServer::reject_query_protocol();
23503        let client = Client::builder(server.base_url())
23504            .timeout(Duration::from_secs(2))
23505            .build()
23506            .expect("client");
23507        let mut worker = Worker::new(client, "rust-workers")
23508            .worker_id("baseline-worker")
23509            .poll_timeout(Duration::from_millis(10));
23510
23511        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
23512            Ok(Value::Null)
23513        });
23514
23515        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
23516        assert_eq!(
23517            server
23518                .worker_protocol_for("/api/worker/workflow-tasks/poll")
23519                .as_deref(),
23520            Some(WORKER_PROTOCOL_VERSION)
23521        );
23522        assert_eq!(
23523            server.worker_protocol_for("/api/worker/query-tasks/poll"),
23524            None,
23525            "a worker without query handlers must not use the query-task endpoint"
23526        );
23527    }
23528
23529    #[tokio::test]
23530    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
23531        let server = MockWorkerServer::reject_query_completion();
23532        let client = Client::builder(server.base_url())
23533            .timeout(Duration::from_secs(2))
23534            .build()
23535            .expect("client");
23536
23537        let error = client
23538            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
23539            .await
23540            .expect_err("expired completion must be rejected");
23541        let Error::QueryFailed(failure) = error else {
23542            panic!("expected typed query failure");
23543        };
23544        assert_eq!(failure.status, 409);
23545        assert_eq!(failure.reason, "query_task_timed_out");
23546
23547        let mut worker = Worker::new(client, "rust-workers")
23548            .worker_id("late-worker")
23549            .poll_timeout(Duration::from_millis(10));
23550        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23551        worker.register_query(
23552            "counter",
23553            "current",
23554            |_ctx, _args| async move { Ok(json!(8)) },
23555        );
23556
23557        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
23558        assert_eq!(
23559            worker
23560                .run_once()
23561                .await
23562                .expect("worker continues after late completion"),
23563            0
23564        );
23565        assert_eq!(
23566            server.request_count("/api/worker/query-tasks/query-late/complete"),
23567            2
23568        );
23569        assert_eq!(
23570            server.request_count("/api/worker/query-tasks/query-late/fail"),
23571            0,
23572            "a server completion rejection must not be reported as an encoding failure"
23573        );
23574    }
23575
23576    #[tokio::test]
23577    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
23578        let server = MockWorkerServer::start();
23579        let client = Client::builder(server.base_url())
23580            .timeout(Duration::from_secs(2))
23581            .build()
23582            .expect("client");
23583        let mut worker = Worker::new(client, "rust-workers")
23584            .worker_id("joined-worker")
23585            .poll_timeout(Duration::from_millis(10));
23586        worker.register_workflow(
23587            "joined.workflow",
23588            |_ctx, _input| async move { Ok(Value::Null) },
23589        );
23590        worker.register_activity(
23591            "joined.activity",
23592            |_ctx, _input| async move { Ok(Value::Null) },
23593        );
23594        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
23595            Ok(Value::Null)
23596        });
23597
23598        worker
23599            .run_until(tokio::time::sleep(Duration::from_millis(20)))
23600            .await
23601            .expect("normal shutdown");
23602
23603        let deregistration_path = "/api/worker/registrations/mock-worker";
23604        assert_eq!(server.request_count(deregistration_path), 1);
23605        for poll_path in [
23606            "/api/worker/workflow-tasks/poll",
23607            "/api/worker/activity-tasks/poll",
23608            "/api/worker/query-tasks/poll",
23609        ] {
23610            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
23611        }
23612        assert_eq!(
23613            server.captured_paths().last().map(String::as_str),
23614            Some(deregistration_path),
23615            "deregistration must start only after every poller has joined"
23616        );
23617    }
23618
23619    #[tokio::test]
23620    async fn registration_failure_does_not_deregister() {
23621        let server = MockWorkerServer::rejected_registration();
23622        let client = Client::builder(server.base_url())
23623            .timeout(Duration::from_secs(2))
23624            .build()
23625            .expect("client");
23626        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
23627
23628        let error = worker
23629            .run_until(async {})
23630            .await
23631            .expect_err("registration must fail");
23632        assert!(matches!(
23633            error,
23634            Error::Http {
23635                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23636                ..
23637            }
23638        ));
23639        assert!(server
23640            .captured_paths()
23641            .iter()
23642            .all(|path| !path.starts_with("/api/worker/registrations/")));
23643    }
23644
23645    #[tokio::test]
23646    async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
23647        let server = MockWorkerServer::rejected_registration_protocol();
23648        let client = Client::builder(server.base_url())
23649            .timeout(Duration::from_secs(2))
23650            .build()
23651            .expect("client");
23652        let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
23653
23654        let error = worker
23655            .run_until(async {})
23656            .await
23657            .expect_err("a protocol 1.16 server must reject this worker");
23658        let Error::Protocol(failure) = error else {
23659            panic!("expected typed protocol rejection");
23660        };
23661        assert_eq!(failure.reason, "unsupported_protocol_version");
23662        assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
23663        assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
23664        assert_eq!(
23665            server
23666                .worker_protocol_for("/api/worker/register")
23667                .as_deref(),
23668            Some(WORKER_PROTOCOL_VERSION)
23669        );
23670    }
23671
23672    #[tokio::test]
23673    async fn declined_registration_does_not_deregister() {
23674        let server = MockWorkerServer::declined_registration();
23675        let client = Client::builder(server.base_url())
23676            .timeout(Duration::from_secs(2))
23677            .build()
23678            .expect("client");
23679        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
23680
23681        let error = worker
23682            .run_until(async {})
23683            .await
23684            .expect_err("declined registration must fail");
23685        assert!(matches!(error, Error::WorkerLoop(_)));
23686        assert!(error.to_string().contains("was not accepted"));
23687        assert!(server
23688            .captured_paths()
23689            .iter()
23690            .all(|path| !path.starts_with("/api/worker/registrations/")));
23691    }
23692
23693    #[tokio::test]
23694    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23695        let server = MockWorkerServer::rejected_deregistration();
23696        let client = Client::builder(server.base_url())
23697            .timeout(Duration::from_secs(2))
23698            .build()
23699            .expect("client");
23700        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23701
23702        let error = worker
23703            .run_until(async {})
23704            .await
23705            .expect_err("deregistration must fail");
23706        assert!(matches!(
23707            error,
23708            Error::Http {
23709                status: reqwest::StatusCode::FORBIDDEN,
23710                ..
23711            }
23712        ));
23713        assert_eq!(
23714            server.request_count("/api/worker/registrations/mock-worker"),
23715            1
23716        );
23717    }
23718
23719    #[tokio::test]
23720    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23721        let server = MockWorkerServer::rejected_deregistration_protocol();
23722        let client = Client::builder(server.base_url())
23723            .timeout(Duration::from_secs(2))
23724            .build()
23725            .expect("client");
23726        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
23727
23728        let error = worker
23729            .run_until(async {})
23730            .await
23731            .expect_err("protocol rejection must fail shutdown");
23732        let Error::Protocol(failure) = error else {
23733            panic!("expected typed protocol failure");
23734        };
23735        assert_eq!(failure.reason, "unsupported_protocol_version");
23736        assert_eq!(
23737            failure.requested_version.as_deref(),
23738            Some(WORKER_PROTOCOL_VERSION)
23739        );
23740        assert_eq!(
23741            server.request_count("/api/worker/registrations/mock-worker"),
23742            1
23743        );
23744    }
23745
23746    #[tokio::test]
23747    async fn primary_poller_error_retains_deregistration_failure_context() {
23748        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
23749        let client = Client::builder(server.base_url())
23750            .timeout(Duration::from_secs(2))
23751            .build()
23752            .expect("client");
23753        let mut worker = Worker::new(client, "rust-workers")
23754            .worker_id("combined-failure")
23755            .poll_timeout(Duration::from_millis(10));
23756        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
23757            Ok(Value::Null)
23758        });
23759
23760        let error = worker
23761            .run()
23762            .await
23763            .expect_err("worker and cleanup must fail");
23764        let summary = error.to_string();
23765        assert!(summary.contains("authentication_failed"));
23766        assert!(summary.contains("worker cannot deregister"));
23767        let Error::WorkerShutdown {
23768            primary,
23769            deregistration,
23770        } = error
23771        else {
23772            panic!("expected combined worker shutdown error");
23773        };
23774        assert!(matches!(
23775            *primary,
23776            Error::Http {
23777                status: reqwest::StatusCode::UNAUTHORIZED,
23778                ..
23779            }
23780        ));
23781        assert!(matches!(
23782            *deregistration,
23783            Error::Http {
23784                status: reqwest::StatusCode::FORBIDDEN,
23785                ..
23786            }
23787        ));
23788        assert_eq!(
23789            server.request_count("/api/worker/registrations/mock-worker"),
23790            1
23791        );
23792    }
23793
23794    #[tokio::test]
23795    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
23796        let server = MockWorkerServer::start();
23797        let client = Client::builder(server.base_url())
23798            .timeout(Duration::from_secs(2))
23799            .build()
23800            .expect("client");
23801        let mut worker = Worker::new(client, "rust-workers")
23802            .worker_id("activity-only-worker")
23803            .poll_timeout(Duration::from_millis(10));
23804
23805        worker.register_activity(
23806            "activity.only",
23807            |_ctx, _args| async move { Ok(Value::Null) },
23808        );
23809
23810        worker.run_until(async {}).await.expect("run worker");
23811    }
23812
23813    #[tokio::test]
23814    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
23815        let server = MockWorkerServer::start();
23816        let client = Client::builder(server.base_url())
23817            .timeout(Duration::from_secs(2))
23818            .build()
23819            .expect("client");
23820        let mut worker = Worker::new(client, "rust-workers")
23821            .worker_id("workflow-only-worker")
23822            .poll_timeout(Duration::from_millis(10));
23823
23824        worker.register_workflow(
23825            "workflow.only",
23826            |_ctx, _input| async move { Ok(Value::Null) },
23827        );
23828
23829        worker.run_until(async {}).await.expect("run worker");
23830    }
23831
23832    #[tokio::test]
23833    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
23834        let server = MockWorkerServer::start();
23835        let client = Client::builder(server.base_url())
23836            .timeout(Duration::from_secs(2))
23837            .build()
23838            .expect("client");
23839        let observations = Arc::new(Mutex::new(Vec::new()));
23840        let observed = Arc::clone(&observations);
23841        let mut worker = Worker::new(client, "rust-workers")
23842            .worker_id("observed-heartbeat-worker")
23843            .poll_timeout(Duration::from_millis(10))
23844            .on_worker_heartbeat(move |observation| {
23845                observed
23846                    .lock()
23847                    .expect("heartbeat observations")
23848                    .push(observation.clone());
23849            });
23850
23851        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
23852            Ok(Value::Null)
23853        });
23854        let acknowledged = Arc::clone(&observations);
23855        worker
23856            .run_until(async move {
23857                tokio::time::timeout(Duration::from_secs(2), async move {
23858                    loop {
23859                        if !acknowledged
23860                            .lock()
23861                            .expect("heartbeat observations")
23862                            .is_empty()
23863                        {
23864                            break;
23865                        }
23866                        tokio::time::sleep(Duration::from_millis(1)).await;
23867                    }
23868                })
23869                .await
23870                .expect("heartbeat acknowledgement within timeout");
23871            })
23872            .await
23873            .expect("run worker");
23874
23875        let observations = observations.lock().expect("heartbeat observations");
23876        let first = observations.first().expect("heartbeat acknowledgement");
23877        assert_eq!(first.worker_id, "observed-heartbeat-worker");
23878        assert_eq!(first.task_queue, "rust-workers");
23879        assert!(first.acknowledged_at_unix_millis > 0);
23880        assert_eq!(first.acknowledgement, json!({}));
23881    }
23882
23883    #[tokio::test]
23884    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
23885        let server = MockWorkerServer::delayed_heartbeat_worker();
23886        let client = Client::builder(server.base_url())
23887            .timeout(Duration::from_secs(3))
23888            .build()
23889            .expect("client");
23890        let observations = Arc::new(Mutex::new(Vec::new()));
23891        let observed = Arc::clone(&observations);
23892        let mut worker = Worker::new(client, "rust-snapshot-workers")
23893            .worker_id("rust-snapshot-worker")
23894            .poll_timeout(Duration::from_millis(10))
23895            .on_worker_heartbeat(move |observation| {
23896                observed
23897                    .lock()
23898                    .expect("heartbeat observations")
23899                    .push(observation.clone());
23900            });
23901
23902        worker.register_workflow("snapshot", |ctx, _input| async move {
23903            ctx.wait_signal("finish").await?;
23904            Ok(json!({"status": "finished"}))
23905        });
23906        worker.register_query("snapshot", "current", |ctx, _args| async move {
23907            Ok(json!(ctx
23908                .signals("increment")
23909                .iter()
23910                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23911                .sum::<i64>()))
23912        });
23913        worker.register_activity("cancel-aware", |_ctx, _args| async move {
23914            Ok(json!({"late": "completion"}))
23915        });
23916
23917        worker
23918            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
23919            .await
23920            .expect("delayed heartbeat must allow a clean worker shutdown");
23921
23922        let observations = observations.lock().expect("heartbeat observations");
23923        assert!(
23924            observations.len() >= 3,
23925            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
23926        );
23927        assert!(
23928            observations.windows(2).all(|pair| {
23929                pair[1].acknowledged_at_unix_millis
23930                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
23931                    >= 850
23932            }),
23933            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
23934        );
23935        drop(observations);
23936
23937        let heartbeat_times = server.request_times("/api/worker/heartbeat");
23938        let delayed_request_at = *heartbeat_times
23939            .get(1)
23940            .expect("intentionally delayed heartbeat request");
23941        let delay_window_start = delayed_request_at + Duration::from_millis(100);
23942        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
23943        for path in [
23944            "/api/worker/workflow-tasks/poll",
23945            "/api/worker/activity-tasks/poll",
23946            "/api/worker/query-tasks/poll",
23947        ] {
23948            assert!(
23949                server
23950                    .request_times(path)
23951                    .iter()
23952                    .any(|received_at| *received_at >= delay_window_start
23953                        && *received_at <= delay_window_end),
23954                "{path} must keep polling while a heartbeat acknowledgement is delayed"
23955            );
23956        }
23957        assert!(
23958            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
23959            "workflow work must be settled"
23960        );
23961        assert!(
23962            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
23963            "activity work must be settled"
23964        );
23965        assert!(
23966            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
23967            "query work must be settled"
23968        );
23969    }
23970
23971    #[tokio::test]
23972    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
23973        let server = MockWorkerServer::heartbeat_retry_worker();
23974        let client = Client::builder(server.base_url())
23975            .timeout(Duration::from_secs(2))
23976            .build()
23977            .expect("client");
23978        let observations = Arc::new(Mutex::new(Vec::new()));
23979        let observed = Arc::clone(&observations);
23980        let worker = Worker::new(client, "rust-workers")
23981            .worker_id("heartbeat-retry-worker")
23982            .retry_policy(WorkerRetryPolicy {
23983                max_retries: 1,
23984                initial_backoff: Duration::from_millis(300),
23985                max_backoff: Duration::from_millis(300),
23986            })
23987            .on_worker_heartbeat(move |observation| {
23988                observed
23989                    .lock()
23990                    .expect("heartbeat observations")
23991                    .push(observation.clone());
23992            });
23993
23994        worker
23995            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
23996            .await
23997            .expect("retryable heartbeat failure must remain bounded and recover");
23998
23999        let observations = observations.lock().expect("heartbeat observations");
24000        assert!(observations.len() >= 3, "heartbeat retry must recover");
24001        assert!(
24002            observations.windows(2).all(|pair| {
24003                pair[1]
24004                    .acknowledged_at_unix_millis
24005                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
24006                    >= 850
24007            }),
24008            "a successful retry must start a fresh advertised cadence: {observations:?}"
24009        );
24010        assert_eq!(
24011            server.request_count("/api/worker/heartbeat"),
24012            observations.len() + 1,
24013            "one retryable failure must add exactly one bounded request"
24014        );
24015    }
24016
24017    #[tokio::test]
24018    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
24019        let server = MockWorkerServer::waiting_query_worker();
24020        let client = Client::builder(server.base_url())
24021            .timeout(Duration::from_secs(2))
24022            .build()
24023            .expect("client");
24024        let observations = Arc::new(Mutex::new(Vec::new()));
24025        let observed = Arc::clone(&observations);
24026        let mut worker = Worker::new(client, "rust-snapshot-workers")
24027            .worker_id("rust-snapshot-worker")
24028            .poll_timeout(Duration::from_millis(10))
24029            .on_worker_heartbeat(move |observation| {
24030                observed
24031                    .lock()
24032                    .expect("heartbeat observations")
24033                    .push(observation.clone());
24034            });
24035
24036        worker.register_workflow("snapshot", |ctx, _input| async move {
24037            ctx.wait_signal("finish").await?;
24038            Ok(json!({"status": "finished"}))
24039        });
24040        worker.register_query("snapshot", "current", |ctx, _args| async move {
24041            let current = ctx
24042                .signals("increment")
24043                .iter()
24044                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24045                .sum::<i64>();
24046            Ok(json!(current))
24047        });
24048        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24049
24050        worker
24051            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24052            .await
24053            .expect("pending workflow and query poller must remain live until shutdown");
24054
24055        assert!(
24056            observations.lock().expect("heartbeat observations").len() >= 4,
24057            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24058        );
24059        assert!(
24060            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24061            "workflow polling must continue after empty replay acknowledgements"
24062        );
24063        assert!(
24064            server.request_count("/api/worker/query-tasks/poll") >= 2,
24065            "query polling must continue after serving the current query"
24066        );
24067        assert_eq!(
24068            server.request_body("/api/worker/register")["capabilities"],
24069            json!([
24070                CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24071                DURABLE_SELECTION_CAPABILITY,
24072                MEMO_UPSERTS_CAPABILITY,
24073                TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24074                QUERY_TASKS_CAPABILITY,
24075                WORKFLOW_UPDATES_CAPABILITY,
24076                MESSAGE_STREAMS_CAPABILITY
24077            ])
24078        );
24079        assert_eq!(
24080            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24081            json!({
24082                "queries": ["current"],
24083                "query_contracts": [],
24084                "signals": [],
24085                "signal_contracts": [],
24086                "updates": ["replace"],
24087                "update_contracts": [],
24088                "update_validators": [],
24089            })
24090        );
24091
24092        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24093        assert_eq!(
24094            opened["commands"],
24095            json!([{
24096                "type": "open_signal_wait",
24097                "signal_name": "finish",
24098            }])
24099        );
24100
24101        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24102            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24103            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24104            let failure = server.request_body(&fail_path);
24105            assert_eq!(
24106                failure["failure"]["type"],
24107                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24108            );
24109            assert_eq!(server.request_count(&completion_path), 0);
24110        }
24111
24112        let query_completion =
24113            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24114        assert_eq!(query_completion["result"], json!(8));
24115
24116        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24117        assert_eq!(
24118            server.request_count(terminal_path),
24119            1,
24120            "the matching signal must settle the workflow exactly once"
24121        );
24122        let terminal = server.request_body(terminal_path);
24123        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24124        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24125        assert_eq!(
24126            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24127                .expect("terminal workflow result"),
24128            json!({"status": "finished"})
24129        );
24130    }
24131
24132    #[tokio::test]
24133    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24134        let server = MockWorkerServer::transient_worker_failures();
24135        let client = Client::builder(server.base_url())
24136            .timeout(Duration::from_secs(2))
24137            .build()
24138            .expect("client");
24139        let mut worker = Worker::new(client, "rust-workers")
24140            .worker_id("retry-worker")
24141            .poll_timeout(Duration::from_millis(10))
24142            .retry_policy(WorkerRetryPolicy {
24143                max_retries: 2,
24144                initial_backoff: Duration::from_millis(1),
24145                max_backoff: Duration::from_millis(1),
24146            });
24147        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24148        worker.register_activity(
24149            "counter.activity",
24150            |_ctx, _input| async move { Ok(Value::Null) },
24151        );
24152        worker.register_query(
24153            "counter",
24154            "current",
24155            |_ctx, _args| async move { Ok(json!(8)) },
24156        );
24157
24158        worker
24159            .run_until(tokio::time::sleep(Duration::from_millis(75)))
24160            .await
24161            .expect("transient failures must not stop the worker");
24162
24163        for path in [
24164            "/api/worker/heartbeat",
24165            "/api/worker/workflow-tasks/poll",
24166            "/api/worker/activity-tasks/poll",
24167            "/api/worker/query-tasks/poll",
24168        ] {
24169            assert!(
24170                server.request_count(path) >= 2,
24171                "{path} must continue after its transient failure"
24172            );
24173        }
24174    }
24175
24176    #[tokio::test]
24177    async fn worker_continues_after_long_poll_capacity_backpressure() {
24178        let server = MockWorkerServer::capacity_limited_activity_poll();
24179        let client = Client::builder(server.base_url())
24180            .timeout(Duration::from_secs(2))
24181            .build()
24182            .expect("client");
24183        let mut worker = Worker::new(client, "rust-workers")
24184            .worker_id("capacity-worker")
24185            .poll_timeout(Duration::from_millis(10))
24186            .retry_policy(WorkerRetryPolicy {
24187                max_retries: 0,
24188                initial_backoff: Duration::from_millis(1),
24189                max_backoff: Duration::from_millis(1),
24190            });
24191        worker.register_activity("capacity.activity", |_ctx, _input| async move {
24192            Ok(json!({"handled": true}))
24193        });
24194
24195        worker
24196            .run_until(tokio::time::sleep(Duration::from_millis(50)))
24197            .await
24198            .expect("capacity backpressure must not stop the worker");
24199
24200        assert!(
24201            server.request_count("/api/worker/activity-tasks/poll") >= 2,
24202            "the activity poller must continue after capacity backpressure"
24203        );
24204        assert_eq!(
24205            server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24206            1,
24207            "the worker must complete work returned after capacity recovers"
24208        );
24209    }
24210
24211    #[test]
24212    fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
24213        let capacity = Error::Http {
24214            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24215            body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
24216        };
24217        assert_eq!(
24218            worker_poll_capacity_retry_after(&capacity),
24219            Some(Duration::from_secs(3))
24220        );
24221
24222        let rejected_capacity = Error::Http {
24223            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24224            body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
24225        };
24226        assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
24227        assert!(!worker_operation_is_retryable(&rejected_capacity));
24228
24229        let ordinary_rate_limit = Error::Http {
24230            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24231            body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
24232                .to_string(),
24233        };
24234        assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
24235        assert!(worker_operation_is_retryable(&ordinary_rate_limit));
24236    }
24237
24238    #[tokio::test]
24239    async fn worker_bounds_transport_retries() {
24240        let server = MockWorkerServer::unavailable_polls();
24241        let client = Client::builder(server.base_url())
24242            .timeout(Duration::from_secs(2))
24243            .build()
24244            .expect("client");
24245        let mut worker = Worker::new(client, "rust-workers")
24246            .worker_id("bounded-retry-worker")
24247            .poll_timeout(Duration::from_millis(10))
24248            .retry_policy(WorkerRetryPolicy {
24249                max_retries: 2,
24250                initial_backoff: Duration::from_millis(1),
24251                max_backoff: Duration::from_millis(1),
24252            });
24253        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24254
24255        let error = worker.run().await.expect_err("retry bound must terminate");
24256        assert!(matches!(error, Error::Transport(_)));
24257        assert_eq!(
24258            server.request_count("/api/worker/workflow-tasks/poll"),
24259            3,
24260            "one initial request plus exactly two retries"
24261        );
24262    }
24263
24264    #[tokio::test]
24265    async fn worker_retry_policy_can_disable_poll_retries() {
24266        let server = MockWorkerServer::unavailable_polls();
24267        let client = Client::builder(server.base_url())
24268            .timeout(Duration::from_secs(2))
24269            .build()
24270            .expect("client");
24271        let mut worker = Worker::new(client, "rust-workers")
24272            .worker_id("no-retry-worker")
24273            .poll_timeout(Duration::from_millis(10))
24274            .retry_policy(WorkerRetryPolicy {
24275                max_retries: 0,
24276                initial_backoff: Duration::from_millis(1),
24277                max_backoff: Duration::from_millis(1),
24278            });
24279        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24280
24281        let error = worker
24282            .run_once()
24283            .await
24284            .expect_err("disabled retries must return the first transport failure");
24285        assert!(matches!(error, Error::Transport(_)));
24286        assert_eq!(
24287            server.request_count("/api/worker/workflow-tasks/poll"),
24288            1,
24289            "max_retries=0 must send only the initial request"
24290        );
24291    }
24292
24293    #[tokio::test]
24294    async fn worker_does_not_retry_authentication_failures() {
24295        let server = MockWorkerServer::unauthorized_polls();
24296        let client = Client::builder(server.base_url())
24297            .timeout(Duration::from_secs(2))
24298            .build()
24299            .expect("client");
24300        let mut worker = Worker::new(client, "rust-workers")
24301            .worker_id("unauthorized-worker")
24302            .poll_timeout(Duration::from_millis(10));
24303        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24304
24305        let error = worker
24306            .run()
24307            .await
24308            .expect_err("authentication must terminate");
24309        let Error::Http { status, body } = error else {
24310            panic!("expected stable HTTP authentication error");
24311        };
24312        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
24313        assert!(body.contains("authentication_failed"));
24314        assert_eq!(
24315            server.request_count("/api/worker/workflow-tasks/poll"),
24316            1,
24317            "authentication failures must not be retried"
24318        );
24319    }
24320
24321    #[derive(Clone, Debug)]
24322    struct CapturedRequest {
24323        method: String,
24324        path: String,
24325        authorization: Option<String>,
24326        namespace: Option<String>,
24327        worker_protocol: Option<String>,
24328        control_protocol: Option<String>,
24329        body: String,
24330        received_at: Instant,
24331    }
24332
24333    struct MockWorkerServer {
24334        addr: SocketAddr,
24335        stop: Arc<AtomicBool>,
24336        requests: Arc<Mutex<Vec<CapturedRequest>>>,
24337        thread: Option<thread::JoinHandle<()>>,
24338    }
24339
24340    #[derive(Clone, Copy, Default)]
24341    struct MockWorkerBehavior {
24342        response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
24343        storage_refusals: usize,
24344        storage_path: Option<&'static str>,
24345        storage_unavailable: bool,
24346        storage_mid_poll: bool,
24347        storage_activity: bool,
24348        storage_query: bool,
24349        storage_wrong_poll_id: bool,
24350        reject_query_protocol: bool,
24351        reject_query_completion: bool,
24352        waiting_query_worker: bool,
24353        decline_registration: bool,
24354        complete_named_signal: bool,
24355        poll_failures_per_path: usize,
24356        long_poll_capacity_responses_per_path: usize,
24357        heartbeat_failures: usize,
24358        heartbeat_failure_request: Option<usize>,
24359        delayed_heartbeat_request: Option<usize>,
24360        heartbeat_response_delay: Duration,
24361        concurrent_requests: bool,
24362        unauthorized_polls: bool,
24363        reject_registration: bool,
24364        reject_registration_protocol: bool,
24365        reject_deregistration: bool,
24366        reject_deregistration_protocol: bool,
24367        cancelled_activity: bool,
24368        draining_polls: bool,
24369        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
24370        workflow_completion_status: Option<&'static str>,
24371        workflow_completion_body: Option<&'static str>,
24372    }
24373
24374    impl MockWorkerServer {
24375        fn start() -> Self {
24376            Self::start_with_behavior(MockWorkerBehavior::default())
24377        }
24378
24379        fn reject_query_protocol() -> Self {
24380            Self::start_with_behavior(MockWorkerBehavior {
24381                reject_query_protocol: true,
24382                ..MockWorkerBehavior::default()
24383            })
24384        }
24385
24386        fn reject_query_completion() -> Self {
24387            Self::start_with_behavior(MockWorkerBehavior {
24388                reject_query_completion: true,
24389                ..MockWorkerBehavior::default()
24390            })
24391        }
24392
24393        fn waiting_query_worker() -> Self {
24394            Self::start_with_behavior(MockWorkerBehavior {
24395                waiting_query_worker: true,
24396                complete_named_signal: true,
24397                ..MockWorkerBehavior::default()
24398            })
24399        }
24400
24401        fn transient_worker_failures() -> Self {
24402            Self::start_with_behavior(MockWorkerBehavior {
24403                poll_failures_per_path: 1,
24404                heartbeat_failures: 1,
24405                ..MockWorkerBehavior::default()
24406            })
24407        }
24408
24409        fn consecutive_poll_failures(count: usize) -> Self {
24410            Self::start_with_behavior(MockWorkerBehavior {
24411                poll_failures_per_path: count,
24412                ..MockWorkerBehavior::default()
24413            })
24414        }
24415
24416        fn capacity_limited_activity_poll() -> Self {
24417            Self::start_with_behavior(MockWorkerBehavior {
24418                long_poll_capacity_responses_per_path: 1,
24419                ..MockWorkerBehavior::default()
24420            })
24421        }
24422
24423        fn delayed_heartbeat_worker() -> Self {
24424            Self::start_with_behavior(MockWorkerBehavior {
24425                waiting_query_worker: true,
24426                delayed_heartbeat_request: Some(2),
24427                heartbeat_response_delay: Duration::from_millis(1_500),
24428                concurrent_requests: true,
24429                cancelled_activity: true,
24430                ..MockWorkerBehavior::default()
24431            })
24432        }
24433
24434        fn heartbeat_retry_worker() -> Self {
24435            Self::start_with_behavior(MockWorkerBehavior {
24436                waiting_query_worker: true,
24437                heartbeat_failure_request: Some(2),
24438                concurrent_requests: true,
24439                ..MockWorkerBehavior::default()
24440            })
24441        }
24442
24443        fn unavailable_polls() -> Self {
24444            Self::start_with_behavior(MockWorkerBehavior {
24445                poll_failures_per_path: usize::MAX,
24446                ..MockWorkerBehavior::default()
24447            })
24448        }
24449
24450        fn unauthorized_polls() -> Self {
24451            Self::start_with_behavior(MockWorkerBehavior {
24452                unauthorized_polls: true,
24453                ..MockWorkerBehavior::default()
24454            })
24455        }
24456
24457        fn rejected_registration() -> Self {
24458            Self::start_with_behavior(MockWorkerBehavior {
24459                reject_registration: true,
24460                ..MockWorkerBehavior::default()
24461            })
24462        }
24463
24464        fn rejected_registration_protocol() -> Self {
24465            Self::start_with_behavior(MockWorkerBehavior {
24466                reject_registration_protocol: true,
24467                ..MockWorkerBehavior::default()
24468            })
24469        }
24470
24471        fn declined_registration() -> Self {
24472            Self::start_with_behavior(MockWorkerBehavior {
24473                decline_registration: true,
24474                ..MockWorkerBehavior::default()
24475            })
24476        }
24477
24478        fn rejected_deregistration() -> Self {
24479            Self::start_with_behavior(MockWorkerBehavior {
24480                reject_deregistration: true,
24481                ..MockWorkerBehavior::default()
24482            })
24483        }
24484
24485        fn rejected_deregistration_protocol() -> Self {
24486            Self::start_with_behavior(MockWorkerBehavior {
24487                reject_deregistration_protocol: true,
24488                ..MockWorkerBehavior::default()
24489            })
24490        }
24491
24492        fn unauthorized_polls_and_rejected_deregistration() -> Self {
24493            Self::start_with_behavior(MockWorkerBehavior {
24494                unauthorized_polls: true,
24495                reject_deregistration: true,
24496                ..MockWorkerBehavior::default()
24497            })
24498        }
24499
24500        fn cancelled_activity() -> Self {
24501            Self::start_with_behavior(MockWorkerBehavior {
24502                cancelled_activity: true,
24503                ..MockWorkerBehavior::default()
24504            })
24505        }
24506
24507        fn draining_polls() -> Self {
24508            Self::start_with_behavior(MockWorkerBehavior {
24509                draining_polls: true,
24510                ..MockWorkerBehavior::default()
24511            })
24512        }
24513
24514        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
24515            Self::start_with_behavior(MockWorkerBehavior {
24516                invalid_task_payload_codec: Some(codec),
24517                ..MockWorkerBehavior::default()
24518            })
24519        }
24520
24521        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
24522            Self::start_with_behavior(MockWorkerBehavior {
24523                workflow_completion_status: Some(status),
24524                workflow_completion_body: Some(body),
24525                ..MockWorkerBehavior::default()
24526            })
24527        }
24528
24529        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
24530            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
24531            listener
24532                .set_nonblocking(true)
24533                .expect("configure mock listener");
24534            let addr = listener.local_addr().expect("mock server address");
24535            let stop = Arc::new(AtomicBool::new(false));
24536            let server_stop = Arc::clone(&stop);
24537            let requests = Arc::new(Mutex::new(Vec::new()));
24538            let server_requests = Arc::clone(&requests);
24539            let thread = thread::spawn(move || {
24540                let mut request_threads = Vec::new();
24541                while !server_stop.load(Ordering::SeqCst) {
24542                    match listener.accept() {
24543                        Ok((mut stream, _)) => {
24544                            if behavior.concurrent_requests {
24545                                let requests = Arc::clone(&server_requests);
24546                                request_threads.push(thread::spawn(move || {
24547                                    handle_mock_worker_request(&mut stream, &requests, behavior)
24548                                }));
24549                            } else {
24550                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
24551                            }
24552                        }
24553                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
24554                            let mut index = 0;
24555                            while index < request_threads.len() {
24556                                if request_threads[index].is_finished() {
24557                                    request_threads
24558                                        .swap_remove(index)
24559                                        .join()
24560                                        .expect("join mock request");
24561                                } else {
24562                                    index += 1;
24563                                }
24564                            }
24565                            thread::sleep(Duration::from_millis(5));
24566                        }
24567                        Err(_) => break,
24568                    }
24569                }
24570                for request_thread in request_threads {
24571                    request_thread.join().expect("join mock request");
24572                }
24573            });
24574
24575            Self {
24576                addr,
24577                stop,
24578                requests,
24579                thread: Some(thread),
24580            }
24581        }
24582
24583        fn base_url(&self) -> String {
24584            format!("http://{}", self.addr)
24585        }
24586
24587        fn worker_protocol_for(&self, path: &str) -> Option<String> {
24588            self.requests
24589                .lock()
24590                .expect("captured requests")
24591                .iter()
24592                .find(|request| request.path == path)
24593                .and_then(|request| request.worker_protocol.clone())
24594        }
24595
24596        fn control_protocol_for(&self, path: &str) -> Option<String> {
24597            self.requests
24598                .lock()
24599                .expect("captured requests")
24600                .iter()
24601                .find(|request| request.path == path)
24602                .and_then(|request| request.control_protocol.clone())
24603        }
24604
24605        fn method_for(&self, path: &str) -> Option<String> {
24606            self.requests
24607                .lock()
24608                .expect("captured requests")
24609                .iter()
24610                .find(|request| request.path == path)
24611                .map(|request| request.method.clone())
24612        }
24613
24614        fn authorization_for(&self, path: &str) -> Option<String> {
24615            self.requests
24616                .lock()
24617                .expect("captured requests")
24618                .iter()
24619                .find(|request| request.path == path)
24620                .and_then(|request| request.authorization.clone())
24621        }
24622
24623        fn namespace_for(&self, path: &str) -> Option<String> {
24624            self.requests
24625                .lock()
24626                .expect("captured requests")
24627                .iter()
24628                .find(|request| request.path == path)
24629                .and_then(|request| request.namespace.clone())
24630        }
24631
24632        fn request_count(&self, path: &str) -> usize {
24633            self.requests
24634                .lock()
24635                .expect("captured requests")
24636                .iter()
24637                .filter(|request| request.path == path)
24638                .count()
24639        }
24640
24641        fn captured_paths(&self) -> Vec<String> {
24642            self.requests
24643                .lock()
24644                .expect("captured requests")
24645                .iter()
24646                .map(|request| request.path.clone())
24647                .collect()
24648        }
24649
24650        fn request_times(&self, path: &str) -> Vec<Instant> {
24651            self.requests
24652                .lock()
24653                .expect("captured requests")
24654                .iter()
24655                .filter(|request| request.path == path)
24656                .map(|request| request.received_at)
24657                .collect()
24658        }
24659
24660        fn request_body(&self, path: &str) -> Value {
24661            let requests = self.requests.lock().expect("captured requests");
24662            let body = &requests
24663                .iter()
24664                .find(|request| request.path == path)
24665                .unwrap_or_else(|| panic!("missing request for {path}"))
24666                .body;
24667            serde_json::from_str(body).unwrap_or_else(|error| {
24668                panic!("invalid JSON request body for {path}: {error}: {body:?}")
24669            })
24670        }
24671
24672        fn request_bodies(&self, path: &str) -> Vec<Value> {
24673            self.requests
24674                .lock()
24675                .expect("captured requests")
24676                .iter()
24677                .filter(|request| request.path == path)
24678                .map(|request| {
24679                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
24680                        panic!(
24681                            "invalid JSON request body for {path}: {error}: {:?}",
24682                            request.body
24683                        )
24684                    })
24685                })
24686                .collect()
24687        }
24688    }
24689
24690    impl Drop for MockWorkerServer {
24691        fn drop(&mut self) {
24692            self.stop.store(true, Ordering::SeqCst);
24693            let _ = TcpStream::connect(self.addr);
24694
24695            if let Some(thread) = self.thread.take() {
24696                thread.join().expect("join mock server");
24697            }
24698        }
24699    }
24700
24701    fn handle_mock_worker_request(
24702        stream: &mut TcpStream,
24703        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
24704        behavior: MockWorkerBehavior,
24705    ) {
24706        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
24707        let mut buffer = [0_u8; 8192];
24708        let mut request = Vec::new();
24709
24710        loop {
24711            match stream.read(&mut buffer) {
24712                Ok(0) => break,
24713                Ok(read) => {
24714                    request.extend_from_slice(&buffer[..read]);
24715                    if mock_request_is_complete(&request) {
24716                        break;
24717                    }
24718                }
24719                Err(error)
24720                    if matches!(
24721                        error.kind(),
24722                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
24723                    ) =>
24724                {
24725                    break;
24726                }
24727                Err(_) => return,
24728            }
24729        }
24730
24731        let request = String::from_utf8_lossy(&request);
24732        let body = request
24733            .split_once("\r\n\r\n")
24734            .map(|(_, body)| body)
24735            .unwrap_or_default();
24736        let path = request
24737            .lines()
24738            .next()
24739            .and_then(|line| line.split_whitespace().nth(1))
24740            .unwrap_or_default();
24741        let method = request
24742            .lines()
24743            .next()
24744            .and_then(|line| line.split_whitespace().next())
24745            .unwrap_or_default();
24746        let authorization = request.lines().find_map(|line| {
24747            let (name, value) = line.split_once(':')?;
24748            name.eq_ignore_ascii_case("Authorization")
24749                .then(|| value.trim().to_string())
24750        });
24751        let namespace = request.lines().find_map(|line| {
24752            let (name, value) = line.split_once(':')?;
24753            name.eq_ignore_ascii_case("X-Namespace")
24754                .then(|| value.trim().to_string())
24755        });
24756        let worker_protocol = request.lines().find_map(|line| {
24757            let (name, value) = line.split_once(':')?;
24758            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
24759                .then(|| value.trim().to_string())
24760        });
24761        let control_protocol = request.lines().find_map(|line| {
24762            let (name, value) = line.split_once(':')?;
24763            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
24764                .then(|| value.trim().to_string())
24765        });
24766        let request_number = {
24767            let mut requests = requests.lock().expect("captured requests");
24768            requests.push(CapturedRequest {
24769                method: method.to_string(),
24770                path: path.to_string(),
24771                authorization,
24772                namespace,
24773                worker_protocol: worker_protocol.clone(),
24774                control_protocol,
24775                body: body.to_string(),
24776                received_at: Instant::now(),
24777            });
24778            requests
24779                .iter()
24780                .filter(|request| request.path == path)
24781                .count()
24782        };
24783
24784        if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
24785            write_mock_response(stream, response.0, &response.1);
24786            return;
24787        }
24788        if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
24789            return;
24790        }
24791        let pressure_path = behavior
24792            .storage_path
24793            .is_some_and(|part| path.contains(part));
24794        let prior_failures = if path.ends_with("/poll") {
24795            behavior.poll_failures_per_path
24796        } else {
24797            0
24798        };
24799        if pressure_path
24800            && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
24801        {
24802            let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
24803            let poll_id = path
24804                .ends_with("/poll")
24805                .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
24806            let mut refusal = storage_refusal(
24807                poll_id,
24808                behavior.storage_unavailable,
24809                behavior.storage_mid_poll,
24810            );
24811            if behavior.storage_wrong_poll_id {
24812                refusal["poll_request_id"] = json!("wrong-poll");
24813            }
24814            write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
24815            return;
24816        }
24817        if path.contains("/storage-task/")
24818            || path.contains("/storage-activity/")
24819            || path.contains("/storage-query/")
24820        {
24821            write_mock_response(stream, "200 OK", "{}");
24822            return;
24823        }
24824        if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
24825            write_mock_response(stream, "200 OK", &json!({"task":{
24826                "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
24827                "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
24828                "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24829                "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24830                "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
24831            }}).to_string());
24832            return;
24833        }
24834        if behavior.storage_activity
24835            && path == "/api/worker/activity-tasks/poll"
24836            && request_number == 1
24837        {
24838            write_mock_response(stream, "200 OK", &json!({"task":{
24839                "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
24840                "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24841                "attempt_number":7, "lease_owner":"storage-worker"
24842            }}).to_string());
24843            return;
24844        }
24845
24846        if path == "/api/worker/register" {
24847            if behavior.reject_registration_protocol {
24848                write_mock_response(
24849                    stream,
24850                    "400 Bad Request",
24851                    r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
24852                );
24853                return;
24854            }
24855            if behavior.reject_registration {
24856                write_mock_response(
24857                    stream,
24858                    "503 Service Unavailable",
24859                    r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
24860                );
24861                return;
24862            }
24863        }
24864
24865        if path.starts_with("/api/worker/registrations/") {
24866            if behavior.reject_deregistration_protocol {
24867                write_mock_response(
24868                    stream,
24869                    "400 Bad Request",
24870                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
24871                );
24872            } else if behavior.reject_deregistration {
24873                write_mock_response(
24874                    stream,
24875                    "403 Forbidden",
24876                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
24877                );
24878            } else {
24879                write_mock_response(
24880                    stream,
24881                    "200 OK",
24882                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
24883                );
24884            }
24885            return;
24886        }
24887
24888        let is_poll = matches!(
24889            path,
24890            "/api/worker/workflow-tasks/poll"
24891                | "/api/worker/activity-tasks/poll"
24892                | "/api/worker/query-tasks/poll"
24893        );
24894        if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
24895            write_mock_response(
24896                stream,
24897                "429 Too Many Requests",
24898                r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
24899            );
24900            return;
24901        }
24902        if is_poll && request_number <= behavior.poll_failures_per_path {
24903            return;
24904        }
24905        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
24906            return;
24907        }
24908        if path == "/api/worker/heartbeat"
24909            && behavior.heartbeat_failure_request == Some(request_number)
24910        {
24911            return;
24912        }
24913        if path == "/api/worker/heartbeat"
24914            && behavior.delayed_heartbeat_request == Some(request_number)
24915        {
24916            thread::sleep(behavior.heartbeat_response_delay);
24917        }
24918        if behavior.unauthorized_polls && is_poll {
24919            write_mock_response(
24920                stream,
24921                "401 Unauthorized",
24922                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
24923            );
24924            return;
24925        }
24926        if behavior.draining_polls && is_poll {
24927            write_mock_response(
24928                stream,
24929                "409 Conflict",
24930                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
24931            );
24932            return;
24933        }
24934
24935        if let Some(codec_case) = behavior.invalid_task_payload_codec {
24936            if is_poll && request_number == 1 {
24937                let mut task = match path {
24938                    "/api/worker/workflow-tasks/poll" => json!({
24939                        "task_id": "codec-workflow",
24940                        "workflow_type": "codec.workflow",
24941                        "payload_codec": DEFAULT_CODEC,
24942                        "workflow_task_attempt": 1,
24943                        "lease_owner": "codec-worker"
24944                    }),
24945                    "/api/worker/activity-tasks/poll" => json!({
24946                        "task_id": "codec-activity",
24947                        "activity_attempt_id": "codec-activity-attempt",
24948                        "activity_type": "codec.activity",
24949                        "payload_codec": DEFAULT_CODEC,
24950                        "attempt_number": 1,
24951                        "lease_owner": "codec-worker"
24952                    }),
24953                    "/api/worker/query-tasks/poll" => json!({
24954                        "query_task_id": "codec-query",
24955                        "query_task_attempt": 1,
24956                        "workflow_type": "codec.workflow",
24957                        "query_name": "known",
24958                        "payload_codec": DEFAULT_CODEC,
24959                        "lease_owner": "codec-worker"
24960                    }),
24961                    _ => unreachable!("is_poll limits task codec probe paths"),
24962                };
24963                codec_case.apply(&mut task);
24964                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
24965                return;
24966            }
24967
24968            if matches!(
24969                path,
24970                "/api/worker/workflow-tasks/codec-workflow/fail"
24971                    | "/api/worker/activity-tasks/codec-activity/fail"
24972                    | "/api/worker/query-tasks/codec-query/fail"
24973            ) {
24974                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
24975                return;
24976            }
24977        }
24978
24979        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
24980            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
24981            let body = format!(
24982                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
24983            );
24984            write_mock_response(stream, "400 Bad Request", &body);
24985            return;
24986        }
24987
24988        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
24989        {
24990            write_mock_response(
24991                stream,
24992                "409 Conflict",
24993                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
24994            );
24995            return;
24996        }
24997
24998        if behavior.workflow_completion_status.is_some()
24999            && path == "/api/worker/workflow-tasks/poll"
25000            && request_number == 1
25001        {
25002            write_mock_response(
25003                stream,
25004                "200 OK",
25005                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
25006            );
25007            return;
25008        }
25009
25010        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
25011            if let (Some(status), Some(body)) = (
25012                behavior.workflow_completion_status,
25013                behavior.workflow_completion_body,
25014            ) {
25015                write_mock_response(stream, status, body);
25016                return;
25017            }
25018        }
25019
25020        if behavior.waiting_query_worker {
25021            if behavior.complete_named_signal
25022                && path == "/api/worker/workflow-tasks/poll"
25023                && request_number == 1
25024            {
25025                let body = json!({
25026                    "task": {
25027                        "task_id": "snapshot-open",
25028                        "workflow_id": "snapshot-1",
25029                        "run_id": "snapshot-run-1",
25030                        "workflow_type": "snapshot",
25031                        "payload_codec": DEFAULT_CODEC,
25032                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25033                            .expect("Avro workflow arguments"),
25034                        "history_events": [],
25035                        "workflow_task_attempt": 1,
25036                        "lease_owner": "rust-snapshot-worker"
25037                    }
25038                })
25039                .to_string();
25040                write_mock_response(stream, "200 OK", &body);
25041                return;
25042            }
25043
25044            let signal_request = request_number - usize::from(behavior.complete_named_signal);
25045            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25046            if path == "/api/worker/workflow-tasks/poll"
25047                && signal_request >= 1
25048                && signal_request <= signal_request_limit
25049            {
25050                let finish = behavior.complete_named_signal && signal_request == 3;
25051                let amounts = if signal_request == 1 {
25052                    vec![3]
25053                } else {
25054                    vec![3, 5]
25055                };
25056                let task_id = if signal_request == 1 {
25057                    "snapshot-wait-3"
25058                } else if finish {
25059                    "snapshot-finish"
25060                } else {
25061                    "snapshot-wait-5"
25062                };
25063                let mut history_events = std::iter::once(json!({
25064                    "event_type": "SignalWaitOpened",
25065                    "payload": {"sequence": 1, "signal_name": "finish"}
25066                }))
25067                .chain(amounts.iter().enumerate().map(|(index, amount)| {
25068                    json!({
25069                        "event_type": "SignalReceived",
25070                        "payload": {
25071                            "signal_id": format!("increment-{amount}"),
25072                            "signal_name": "increment",
25073                            "workflow_sequence": index + 2,
25074                            "payload_codec": DEFAULT_CODEC,
25075                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25076                                .expect("Avro signal envelope")
25077                        }
25078                    })
25079                }))
25080                .collect::<Vec<_>>();
25081                let (resume_id, resume_name, resume_arguments) = if finish {
25082                    history_events.push(json!({
25083                        "event_type": "SignalReceived",
25084                        "payload": {
25085                            "signal_id": "finish",
25086                            "signal_name": "finish",
25087                            "workflow_sequence": 4,
25088                            "payload_codec": DEFAULT_CODEC,
25089                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25090                                .expect("Avro finish signal envelope")
25091                        }
25092                    }));
25093                    (
25094                        "finish".to_string(),
25095                        "finish".to_string(),
25096                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
25097                            .expect("Avro finish resume signal"),
25098                    )
25099                } else {
25100                    let amount = amounts.last().expect("amount");
25101                    (
25102                        format!("increment-{amount}"),
25103                        "increment".to_string(),
25104                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25105                            .expect("Avro increment resume signal"),
25106                    )
25107                };
25108                let body = json!({
25109                    "task": {
25110                        "task_id": task_id,
25111                        "workflow_id": "snapshot-1",
25112                        "run_id": "snapshot-run-1",
25113                        "workflow_type": "snapshot",
25114                        "payload_codec": DEFAULT_CODEC,
25115                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25116                            .expect("Avro workflow arguments"),
25117                        "history_events": history_events,
25118                        "workflow_task_attempt": 1,
25119                        "workflow_signal_id": resume_id,
25120                        "signal_name": resume_name,
25121                        "signal_arguments": resume_arguments,
25122                        "lease_owner": "rust-snapshot-worker"
25123                    }
25124                })
25125                .to_string();
25126                write_mock_response(stream, "200 OK", &body);
25127                return;
25128            }
25129
25130            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25131                let history_events = [3, 5]
25132                    .into_iter()
25133                    .enumerate()
25134                    .map(|(index, amount)| {
25135                        json!({
25136                            "event_type": "SignalReceived",
25137                            "payload": {
25138                                "signal_id": format!("increment-{amount}"),
25139                                "signal_name": "increment",
25140                                "workflow_sequence": index + 2,
25141                                "payload_codec": DEFAULT_CODEC,
25142                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25143                                    .expect("Avro query signal envelope")
25144                            }
25145                        })
25146                    })
25147                    .collect::<Vec<_>>();
25148                let body = json!({
25149                    "task": {
25150                        "query_task_id": "snapshot-current",
25151                        "query_task_attempt": 1,
25152                        "lease_owner": "rust-snapshot-worker",
25153                        "workflow_id": "snapshot-1",
25154                        "run_id": "snapshot-run-1",
25155                        "workflow_type": "snapshot",
25156                        "query_name": "current",
25157                        "payload_codec": DEFAULT_CODEC,
25158                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25159                            .expect("Avro workflow arguments"),
25160                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25161                            .expect("Avro query arguments"),
25162                        "history_events": history_events,
25163                        "run_status": "waiting"
25164                    }
25165                })
25166                .to_string();
25167                write_mock_response(stream, "200 OK", &body);
25168                return;
25169            }
25170
25171            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25172                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25173            {
25174                write_mock_response(
25175                    stream,
25176                    "200 OK",
25177                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
25178                );
25179                return;
25180            }
25181
25182            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25183                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25184                return;
25185            }
25186
25187            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25188                write_mock_response(
25189                    stream,
25190                    "200 OK",
25191                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25192                );
25193                return;
25194            }
25195
25196            if path == "/api/worker/query-tasks/snapshot-current/complete" {
25197                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
25198                return;
25199            }
25200        }
25201
25202        if matches!(
25203            path,
25204            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
25205        ) {
25206            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25207                .expect("typed mock result");
25208            let body = json!({
25209                "result": typed_fidelity_probe().into_json().expect("result projection"),
25210                "result_envelope": result,
25211            })
25212            .to_string();
25213            write_mock_response(stream, "200 OK", &body);
25214            return;
25215        }
25216
25217        if path == "/api/workflows/typed-1" {
25218            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25219                .expect("typed mock result");
25220            let body = json!({
25221                "workflow_id": "typed-1",
25222                "run_id": "run-typed-1",
25223                "workflow_type": "typed.echo",
25224                "status": "completed",
25225                "output": typed_fidelity_probe().into_json().expect("output projection"),
25226                "output_envelope": result,
25227            })
25228            .to_string();
25229            write_mock_response(stream, "200 OK", &body);
25230            return;
25231        }
25232
25233        let (status, body) = match path {
25234            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
25235            "/api/workflows" => (
25236                "201 Created",
25237                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
25238            ),
25239            "/api/worker/register" if behavior.decline_registration => (
25240                "200 OK",
25241                r#"{"worker_id":"declined-worker","registered":false}"#,
25242            ),
25243            "/api/worker/register" if behavior.waiting_query_worker => (
25244                "200 OK",
25245                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
25246            ),
25247            "/api/worker/register" => (
25248                "200 OK",
25249                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
25250            ),
25251            "/api/worker/heartbeat" => ("200 OK", "{}"),
25252            "/api/worker/activity-tasks/poll"
25253                if behavior.cancelled_activity && request_number == 1 =>
25254            {
25255                (
25256                    "200 OK",
25257                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
25258                )
25259            }
25260            "/api/worker/activity-tasks/poll"
25261                if behavior.long_poll_capacity_responses_per_path > 0
25262                    && request_number
25263                        == behavior
25264                            .long_poll_capacity_responses_per_path
25265                            .saturating_add(1) =>
25266            {
25267                (
25268                    "200 OK",
25269                    r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
25270                )
25271            }
25272            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
25273                ("200 OK", r#"{"task":null}"#)
25274            }
25275            "/api/worker/query-tasks/poll"
25276                if behavior.reject_query_completion && request_number == 1 =>
25277            {
25278                (
25279                    "200 OK",
25280                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
25281                )
25282            }
25283            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
25284            "/api/worker/query-tasks/query-capture/complete"
25285            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
25286            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
25287                "200 OK",
25288                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
25289            ),
25290            "/api/worker/activity-tasks/activity-cancel/complete" => (
25291                "409 Conflict",
25292                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
25293            ),
25294            "/api/worker/activity-tasks/activity-typed/complete"
25295            | "/api/worker/activity-tasks/activity-typed/fail"
25296            | "/api/worker/activity-tasks/capacity-activity/complete"
25297            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
25298            "/api/workflows/counter-1/query/current" => (
25299                "200 OK",
25300                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
25301            ),
25302            "/api/workflows/counter-1/query/missing" => (
25303                "404 Not Found",
25304                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
25305            ),
25306            "/api/workflows/wf-lifecycle/cancel" => (
25307                "200 OK",
25308                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
25309            ),
25310            "/api/workflows/wf-lifecycle/terminate" => (
25311                "200 OK",
25312                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
25313            ),
25314            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
25315                "200 OK",
25316                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
25317            ),
25318            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
25319                "200 OK",
25320                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
25321            ),
25322            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
25323            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
25324                "409 Conflict",
25325                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
25326            ),
25327            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
25328                "200 OK",
25329                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
25330            ),
25331            "/api/workflows/wf-cancelled" => (
25332                "200 OK",
25333                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
25334            ),
25335            "/api/workflows/wf-terminated" => (
25336                "200 OK",
25337                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
25338            ),
25339            "/api/workflows/wf-timed-out" => (
25340                "200 OK",
25341                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
25342            ),
25343            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
25344                "200 OK",
25345                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
25346            ),
25347            "/api/workflows/wf-selected" => (
25348                "200 OK",
25349                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
25350            ),
25351            "/api/workflows/wf-selected/runs/run-selected" => (
25352                "200 OK",
25353                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
25354            ),
25355            _ => ("404 Not Found", r#"{"message":"not found"}"#),
25356        };
25357        write_mock_response(stream, status, body);
25358    }
25359
25360    fn mock_request_is_complete(request: &[u8]) -> bool {
25361        let Some(header_end) = request
25362            .windows(4)
25363            .position(|window| window == b"\r\n\r\n")
25364            .map(|position| position + 4)
25365        else {
25366            return false;
25367        };
25368        let headers = String::from_utf8_lossy(&request[..header_end]);
25369        let content_length = headers.lines().find_map(|line| {
25370            let (name, value) = line.split_once(':')?;
25371            name.eq_ignore_ascii_case("content-length")
25372                .then(|| value.trim().parse::<usize>().ok())
25373                .flatten()
25374        });
25375
25376        request.len() >= header_end + content_length.unwrap_or(0)
25377    }
25378
25379    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
25380        let response = format!(
25381            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
25382            body.len()
25383        );
25384
25385        let _ = stream.write_all(response.as_bytes());
25386        let _ = stream.flush();
25387    }
25388}