Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod runtime_payloads;
4mod runtime_uploads;
5
6use std::{
7    any::{type_name, Any, TypeId},
8    collections::{BTreeMap, HashMap},
9    future::Future,
10    io::{self, Read},
11    pin::Pin,
12    sync::{
13        atomic::{AtomicBool, Ordering},
14        Arc, Mutex, OnceLock,
15    },
16    task::{Context as TaskContext, Poll},
17    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
18};
19
20use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
21use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
22use chrono::DateTime;
23use futures_util::{future::OptionFuture, task::noop_waker_ref};
24use serde::{
25    de::DeserializeOwned,
26    ser::{SerializeMap, SerializeSeq},
27    Deserialize, Deserializer, Serialize, Serializer,
28};
29pub use serde_json::{json, Value};
30use sha2::{Digest, Sha256};
31use thiserror::Error;
32pub use uuid::Uuid;
33
34pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
35/// First additive worker protocol that defines portable worker-affinity features.
36pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
37pub const CONTROL_PLANE_VERSION: &str = "2";
38pub const DEFAULT_CODEC: &str = "avro";
39pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
40/// Worker-registration capability for authored condition-wait occurrence identity.
41pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
42    "condition_wait_occurrence_identity";
43/// Worker-registration capability for portable memo upserts.
44pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
45/// Worker-registration capability for server-routed read-only queries.
46pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
47/// Worker-registration capability for canonical typed search attributes.
48pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
49/// Worker-registration capability for synchronous workflow updates.
50pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
51/// Worker-registration capability for durable named input streams.
52pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
53/// Worker-registration capability for persisted first-completion selection.
54pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
55pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
56pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
57pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
58pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
59pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
60/// First additive worker protocol that defines query-task transport.
61pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
62/// First additive worker protocol that defines typed search-attribute upserts.
63pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
64/// First additive worker protocol that defines portable memo upserts.
65pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
66/// First additive worker protocol that preserves declared search-attribute types.
67pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
68/// First additive worker protocol that defines external durable condition waits.
69pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
70/// First additive worker protocol that preserves authored condition-wait occurrences.
71pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
72/// First additive worker protocol that defines durable selection groups.
73pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
74
75pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
76    let Some((major, minor)) = version.split_once('.') else {
77        return false;
78    };
79    major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
80}
81
82fn validate_user_signal_name(signal_name: &str) -> Result<()> {
83    if signal_name == MESSAGE_STREAM_SIGNAL {
84        return Err(Error::Codec(format!(
85            "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
86        )));
87    }
88    Ok(())
89}
90
91const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
92const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
93    "Workflow task waiting for scheduled history.";
94const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
95const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
96const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
97const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
98const MAX_MEMO_ENTRIES: usize = 100;
99const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
100const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
101
102const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
103    "lease_expired",
104    "query_task_not_found",
105    "query_task_not_leased",
106    "query_task_timed_out",
107];
108
109/// Truthful service-worker manifest for features this SDK currently refuses.
110pub fn portable_worker_affinity_capability_manifest() -> Value {
111    json!({
112        "local_activities": {
113            "supported": false,
114            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
115            "reason": "rust_worker_does_not_execute_record_local_activity",
116        },
117        "worker_sessions": {
118            "supported": false,
119            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
120            "reason": "rust_worker_has_no_typed_session_lifecycle",
121        },
122        "sticky_execution": {
123            "supported": false,
124            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
125            "reason": "rust_worker_uses_complete_durable_history_replay",
126        },
127    })
128}
129
130/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
131pub const AVRO_VALUE_SCHEMA_JSON: &str =
132    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
133pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
134pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
135const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
136
137static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
138static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
139    OnceLock::new();
140
141#[derive(Clone, Copy)]
142enum RequestProtocol {
143    ControlPlane,
144    Worker(&'static str),
145}
146
147pub type Result<T> = std::result::Result<T, Error>;
148
149#[derive(Debug, Error)]
150pub enum Error {
151    #[error("transport error: {0}")]
152    Transport(#[from] reqwest::Error),
153    #[error(
154        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
155    )]
156    InvalidBaseUrl,
157    #[error("json error: {0}")]
158    Json(#[from] serde_json::Error),
159    #[error("http {status}: {body}")]
160    Http {
161        status: reqwest::StatusCode,
162        body: String,
163    },
164    #[error("codec error: {0}")]
165    Codec(String),
166    #[error(transparent)]
167    QueryFailed(QueryFailure),
168    #[error(transparent)]
169    Protocol(ProtocolFailure),
170    #[error(transparent)]
171    NonDeterministicReplay(ReplayFailure),
172    #[error(transparent)]
173    ChildWorkflowFailed(ChildWorkflowFailure),
174    #[error(transparent)]
175    ActivityFailed(ActivityFailure),
176    #[error(transparent)]
177    ParallelFailed(ParallelFailure),
178    #[error(transparent)]
179    SagaCompensationFailed(SagaCompensationFailure),
180    #[error(transparent)]
181    InvalidParallelGroup(ParallelGroupError),
182    #[error(transparent)]
183    DurableOperationCancelled(DurableOperationCancelled),
184    #[error(transparent)]
185    WorkflowCancellationRequested(WorkflowCancellationRequested),
186    #[error(transparent)]
187    WorkflowCommandRejected(WorkflowCommandRejection),
188    #[error(transparent)]
189    WorkflowFailed(WorkflowTerminalOutcome),
190    #[error(transparent)]
191    WorkflowCancelled(WorkflowTerminalOutcome),
192    #[error(transparent)]
193    WorkflowTerminated(WorkflowTerminalOutcome),
194    #[error(transparent)]
195    WorkflowTimedOut(WorkflowTerminalOutcome),
196    #[error(transparent)]
197    ActivityTaskRejected(ActivityTaskRejection),
198    #[error("workflow handler {0:?} is not registered")]
199    WorkflowNotRegistered(String),
200    #[error("activity handler {0:?} is not registered")]
201    ActivityNotRegistered(String),
202    #[error(
203        "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
204    )]
205    HandlerType {
206        handler_kind: HandlerKind,
207        handler_name: String,
208        value_kind: HandlerValueKind,
209        rust_type: &'static str,
210        message: String,
211    },
212    #[error("workflow future yielded without emitting a durable command")]
213    WorkflowYieldedWithoutCommand,
214    #[error(
215        "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
216    )]
217    MissingWorkflowCommandIdentity,
218    #[error("workflow state lock is poisoned")]
219    WorkflowStatePoisoned,
220    #[error("timer duration is too large for the worker protocol")]
221    TimerDurationOverflow,
222    #[error(transparent)]
223    InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
224    #[error(transparent)]
225    InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
226    #[error("operation timed out")]
227    Timeout,
228    #[error(
229        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
230    )]
231    MissingRoleCredentials {
232        role: &'static str,
233        opposite_role: &'static str,
234    },
235    #[error("worker loop error: {0}")]
236    WorkerLoop(String),
237    #[error(
238        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
239    )]
240    UnsupportedUpdateValidators { workflow_type: String },
241    #[error("{primary}; worker deregistration also failed: {deregistration}")]
242    WorkerShutdown {
243        primary: Box<Error>,
244        deregistration: Box<Error>,
245    },
246    #[error("invalid child workflow options: {0}")]
247    InvalidChildWorkflowOptions(String),
248    #[error("invalid workflow memo update: {0}")]
249    InvalidMemoUpdate(String),
250    #[error(
251        "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
252    )]
253    WorkflowMemoUpdatesUnavailable,
254    #[error(transparent)]
255    InvalidActivityOptions(ActivityOptionsError),
256    #[error(transparent)]
257    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
258    #[doc(hidden)]
259    #[error("workflow requested continue as new")]
260    ContinueAsNew(ContinueAsNewRequest),
261}
262
263/// Validation failure for a durable condition-wait definition.
264#[derive(Clone, Debug, Error, PartialEq, Eq)]
265pub enum ConditionWaitOptionsError {
266    #[error("condition_key must be non-empty")]
267    EmptyKey,
268    #[error("condition_definition_fingerprint must be non-empty")]
269    EmptyPredicateIdentity,
270    #[error("condition timeout is too large for the worker protocol")]
271    TimeoutOverflow,
272}
273
274/// Stable identity and optional durable timeout for a condition wait.
275///
276/// `predicate_identity` is recorded as the worker protocol's
277/// `condition_definition_fingerprint` and must change whenever predicate
278/// behavior changes. Prefer the [`wait_condition!`] macro when the predicate
279/// is written inline; it derives this identity from the predicate tokens.
280#[derive(Clone, Debug, PartialEq, Eq)]
281pub struct ConditionWaitOptions {
282    condition_key: String,
283    predicate_identity: String,
284    timeout: Option<Duration>,
285}
286
287impl ConditionWaitOptions {
288    pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
289        Self {
290            condition_key: condition_key.into(),
291            predicate_identity: predicate_identity.into(),
292            timeout: None,
293        }
294    }
295
296    pub fn timeout(mut self, timeout: Duration) -> Self {
297        self.timeout = Some(timeout);
298        self
299    }
300
301    fn validate(
302        &self,
303    ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
304        let condition_key = self.condition_key.trim();
305        if condition_key.is_empty() {
306            return Err(ConditionWaitOptionsError::EmptyKey);
307        }
308        let predicate_identity = self.predicate_identity.trim();
309        if predicate_identity.is_empty() {
310            return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
311        }
312        let timeout_seconds = self
313            .timeout
314            .map(|timeout| {
315                timeout
316                    .as_secs()
317                    .checked_add(u64::from(timeout.subsec_nanos() > 0))
318                    .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
319            })
320            .transpose()?;
321
322        Ok(ValidatedConditionWaitOptions {
323            condition_key: condition_key.to_string(),
324            predicate_identity: predicate_identity.to_string(),
325            timeout_seconds,
326        })
327    }
328}
329
330#[derive(Clone, Debug, PartialEq, Eq)]
331struct ValidatedConditionWaitOptions {
332    condition_key: String,
333    predicate_identity: String,
334    timeout_seconds: Option<u64>,
335}
336
337const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
338
339/// Unambiguous terminal result of a durable condition wait.
340#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
341#[serde(rename_all = "snake_case")]
342pub enum ConditionWaitResult {
343    Satisfied,
344    TimedOut,
345}
346
347impl ConditionWaitResult {
348    pub fn is_satisfied(self) -> bool {
349        self == Self::Satisfied
350    }
351
352    pub fn is_timed_out(self) -> bool {
353        self == Self::TimedOut
354    }
355}
356
357/// Build the stable condition definition identity used by [`wait_condition!`].
358#[doc(hidden)]
359pub fn __condition_definition_fingerprint(source: &str) -> String {
360    let mut digest = Sha256::new();
361    digest.update(b"durable-workflow-rust.wait-condition.v1\0");
362    digest.update(source.as_bytes());
363    format!("sha256:{:x}", digest.finalize())
364}
365
366/// Create a durable condition wait whose predicate definition is fingerprinted
367/// from its inline Rust tokens.
368///
369/// The returned [`ConditionWaitCall`] must be awaited. The timeout form is
370/// `wait_condition!(ctx, "approval", timeout: duration, || predicate)`.
371#[macro_export]
372macro_rules! wait_condition {
373    ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
374        $ctx.wait_condition(
375            $crate::ConditionWaitOptions::new(
376                $key,
377                $crate::__condition_definition_fingerprint(concat!(
378                    module_path!(),
379                    "\0",
380                    stringify!($predicate)
381                )),
382            )
383            .timeout($timeout),
384            $predicate,
385        )
386    }};
387    ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
388        $ctx.wait_condition(
389            $crate::ConditionWaitOptions::new(
390                $key,
391                $crate::__condition_definition_fingerprint(concat!(
392                    module_path!(),
393                    "\0",
394                    stringify!($predicate)
395                )),
396            ),
397            $predicate,
398        )
399    }};
400}
401
402const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
403const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
404const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
405const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
406const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
407
408/// Validation failure for a typed workflow search-attribute update.
409#[derive(Clone, Debug, Error, PartialEq, Eq)]
410pub enum SearchAttributeUpdateError {
411    #[error("search-attribute update requires at least one attribute")]
412    Empty,
413    #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
414    InvalidKey(String),
415    #[error("search-attribute update exceeds the limit of 100 attributes")]
416    TooManyAttributes,
417    #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
418    ValueTooLong {
419        key: String,
420        kind: &'static str,
421        limit: usize,
422    },
423    #[error(
424        "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
425    )]
426    EmptyString(String),
427    #[error("search attribute {0:?} has a non-finite float value")]
428    NonFiniteFloat(String),
429    #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
430    InvalidDateTime(String),
431    #[error("search-attribute update exceeds the 65536-byte protocol limit")]
432    PayloadTooLarge,
433}
434
435/// One public typed search-attribute value.
436#[derive(Clone, Debug, PartialEq)]
437pub enum SearchAttributeValue {
438    String(String),
439    Keyword(String),
440    KeywordList(Vec<String>),
441    Int(i64),
442    Float(f64),
443    Bool(bool),
444    DateTime(String),
445    Delete,
446}
447
448impl SearchAttributeValue {
449    fn type_name(&self) -> Option<&'static str> {
450        match self {
451            Self::String(_) => Some("string"),
452            Self::Keyword(_) => Some("keyword"),
453            Self::KeywordList(_) => Some("keyword_list"),
454            Self::Int(_) => Some("int"),
455            Self::Float(_) => Some("float"),
456            Self::Bool(_) => Some("bool"),
457            Self::DateTime(_) => Some("datetime"),
458            Self::Delete => None,
459        }
460    }
461
462    fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
463        let normalize_string = |value: String, kind: &'static str, limit: usize| {
464            let value = value.trim().to_string();
465            if value.is_empty() {
466                return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
467            }
468            if value.len() > limit {
469                return Err(SearchAttributeUpdateError::ValueTooLong {
470                    key: key.to_string(),
471                    kind,
472                    limit,
473                });
474            }
475            Ok(value)
476        };
477
478        match self {
479            Self::String(value) => Ok(Self::String(normalize_string(
480                value,
481                "string",
482                MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
483            )?)),
484            Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
485                value,
486                "keyword",
487                MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
488            )?)),
489            Self::KeywordList(values) => {
490                let values = values
491                    .into_iter()
492                    .map(|value| {
493                        let value = value.trim().to_string();
494                        if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
495                            return Err(SearchAttributeUpdateError::ValueTooLong {
496                                key: key.to_string(),
497                                kind: "keyword-list entry",
498                                limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
499                            });
500                        }
501                        Ok(value)
502                    })
503                    .collect::<std::result::Result<Vec<_>, _>>()?;
504                Ok(Self::KeywordList(values))
505            }
506            Self::Float(value) if !value.is_finite() => {
507                Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
508            }
509            Self::DateTime(value) => {
510                let value =
511                    normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
512                if DateTime::parse_from_rfc3339(&value).is_err() {
513                    return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
514                }
515                Ok(Self::DateTime(value))
516            }
517            value => Ok(value),
518        }
519    }
520
521    fn into_json(self) -> Value {
522        match self {
523            Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
524                Value::String(value)
525            }
526            Self::KeywordList(values) => {
527                Value::Array(values.into_iter().map(Value::String).collect())
528            }
529            Self::Int(value) => json!(value),
530            Self::Float(value) => json!(value),
531            Self::Bool(value) => json!(value),
532            Self::Delete => Value::Null,
533        }
534    }
535}
536
537/// Validated typed workflow-side search-attribute mutation.
538#[derive(Clone, Debug, Default, PartialEq)]
539pub struct SearchAttributeUpdate {
540    attributes: BTreeMap<String, SearchAttributeValue>,
541}
542
543impl SearchAttributeUpdate {
544    pub fn new() -> Self {
545        Self::default()
546    }
547
548    pub fn set(
549        mut self,
550        key: impl Into<String>,
551        value: SearchAttributeValue,
552    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
553        let key = key.into();
554        validate_search_attribute_key(&key)?;
555        if !self.attributes.contains_key(&key)
556            && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
557        {
558            return Err(SearchAttributeUpdateError::TooManyAttributes);
559        }
560        self.attributes.insert(key.clone(), value.normalized(&key)?);
561        self.validate_size()?;
562        Ok(self)
563    }
564
565    pub fn string(
566        self,
567        key: impl Into<String>,
568        value: impl Into<String>,
569    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
570        self.set(key, SearchAttributeValue::String(value.into()))
571    }
572
573    pub fn keyword(
574        self,
575        key: impl Into<String>,
576        value: impl Into<String>,
577    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
578        self.set(key, SearchAttributeValue::Keyword(value.into()))
579    }
580
581    pub fn keyword_list<I, V>(
582        self,
583        key: impl Into<String>,
584        values: I,
585    ) -> std::result::Result<Self, SearchAttributeUpdateError>
586    where
587        I: IntoIterator<Item = V>,
588        V: Into<String>,
589    {
590        self.set(
591            key,
592            SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
593        )
594    }
595
596    pub fn int(
597        self,
598        key: impl Into<String>,
599        value: i64,
600    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
601        self.set(key, SearchAttributeValue::Int(value))
602    }
603
604    pub fn float(
605        self,
606        key: impl Into<String>,
607        value: f64,
608    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
609        self.set(key, SearchAttributeValue::Float(value))
610    }
611
612    pub fn bool(
613        self,
614        key: impl Into<String>,
615        value: bool,
616    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
617        self.set(key, SearchAttributeValue::Bool(value))
618    }
619
620    pub fn datetime(
621        self,
622        key: impl Into<String>,
623        value: impl Into<String>,
624    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
625        self.set(key, SearchAttributeValue::DateTime(value.into()))
626    }
627
628    pub fn delete(
629        self,
630        key: impl Into<String>,
631    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
632        self.set(key, SearchAttributeValue::Delete)
633    }
634
635    fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
636        let (attributes, _) = self.clone().into_wire_parts();
637        if serde_json::to_vec(&attributes)
638            .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
639            .unwrap_or(true)
640        {
641            return Err(SearchAttributeUpdateError::PayloadTooLarge);
642        }
643        Ok(())
644    }
645
646    fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
647        let mut attributes = serde_json::Map::new();
648        let mut attribute_types = BTreeMap::new();
649        for (key, value) in self.attributes {
650            if let Some(type_name) = value.type_name() {
651                attribute_types.insert(key.clone(), type_name.to_string());
652            }
653            attributes.insert(key, value.into_json());
654        }
655        (Value::Object(attributes), attribute_types)
656    }
657
658    fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
659        if self.attributes.is_empty() {
660            return Err(SearchAttributeUpdateError::Empty);
661        }
662        self.validate_size()
663    }
664}
665
666fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
667    let valid = !key.is_empty()
668        && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
669        && key
670            .bytes()
671            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
672    if valid {
673        Ok(())
674    } else {
675        Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
676    }
677}
678
679/// The registered handler family reported by [`Error::HandlerType`].
680#[derive(Clone, Copy, Debug, PartialEq, Eq)]
681pub enum HandlerKind {
682    Workflow,
683    Activity,
684}
685
686impl std::fmt::Display for HandlerKind {
687    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
688        formatter.write_str(match self {
689            Self::Workflow => "workflow",
690            Self::Activity => "activity",
691        })
692    }
693}
694
695/// Whether a typed handler failed to adapt its input or result.
696#[derive(Clone, Copy, Debug, PartialEq, Eq)]
697pub enum HandlerValueKind {
698    Input,
699    Result,
700}
701
702impl std::fmt::Display for HandlerValueKind {
703    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
704        formatter.write_str(match self {
705            Self::Input => "input",
706            Self::Result => "result",
707        })
708    }
709}
710
711/// The lifecycle command sent to a workflow execution.
712#[derive(Clone, Copy, Debug, PartialEq, Eq)]
713pub enum WorkflowCommandKind {
714    Cancel,
715    Terminate,
716}
717
718impl WorkflowCommandKind {
719    fn as_str(self) -> &'static str {
720        match self {
721            Self::Cancel => "cancel",
722            Self::Terminate => "terminate",
723        }
724    }
725}
726
727/// Optional structured fields for a cancellation or termination request.
728#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
729pub struct WorkflowCommandOptions {
730    #[serde(skip_serializing_if = "Option::is_none")]
731    pub reason: Option<String>,
732    #[serde(skip_serializing_if = "Option::is_none")]
733    pub request_id: Option<String>,
734}
735
736/// Server-enforced timeout policy for a workflow start.
737///
738/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
739/// only bounds how long the caller waits. A server deadline produces a terminal
740/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
741/// `run_timeout`.
742#[derive(Clone, Debug, PartialEq, Eq)]
743pub struct WorkflowStartOptions {
744    pub execution_timeout_seconds: u64,
745    pub run_timeout_seconds: u64,
746}
747
748impl Default for WorkflowStartOptions {
749    fn default() -> Self {
750        Self {
751            execution_timeout_seconds: 3600,
752            run_timeout_seconds: 600,
753        }
754    }
755}
756
757impl WorkflowStartOptions {
758    pub fn new() -> Self {
759        Self::default()
760    }
761
762    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
763        self.execution_timeout_seconds = seconds;
764        self
765    }
766
767    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
768        self.run_timeout_seconds = seconds;
769        self
770    }
771
772    fn validate(&self) -> Result<()> {
773        if self.execution_timeout_seconds == 0 {
774            return Err(Error::Codec(
775                "execution_timeout_seconds must be at least 1".to_string(),
776            ));
777        }
778        if self.run_timeout_seconds == 0 {
779            return Err(Error::Codec(
780                "run_timeout_seconds must be at least 1".to_string(),
781            ));
782        }
783        if self.run_timeout_seconds > self.execution_timeout_seconds {
784            return Err(Error::Codec(
785                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
786            ));
787        }
788
789        Ok(())
790    }
791}
792
793/// Optional routing overrides for a continue-as-new transition.
794///
795/// Omitted values retain the current workflow type and task queue. Server-owned
796/// instance metadata is not accepted here and is carried by the server.
797#[derive(Clone, Debug, Default, PartialEq, Eq)]
798pub struct ContinueAsNewOptions {
799    pub workflow_type: Option<String>,
800    pub task_queue: Option<String>,
801}
802
803impl ContinueAsNewOptions {
804    pub fn new() -> Self {
805        Self::default()
806    }
807
808    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
809        self.workflow_type = Some(workflow_type.into());
810        self
811    }
812
813    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
814        self.task_queue = Some(task_queue.into());
815        self
816    }
817
818    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
819        for (field, value) in [
820            ("workflow_type", self.workflow_type.as_deref()),
821            ("task_queue", self.task_queue.as_deref()),
822        ] {
823            if value.is_some_and(|value| value.trim().is_empty()) {
824                return Err(ContinueAsNewOptionsError {
825                    field,
826                    message: format!("{field} must not be empty"),
827                });
828            }
829        }
830        Ok(())
831    }
832}
833
834/// A stable validation error raised before a continue-as-new command is emitted.
835#[derive(Clone, Debug, Error, PartialEq, Eq)]
836#[error("invalid continue-as-new option {field}: {message}")]
837pub struct ContinueAsNewOptionsError {
838    pub field: &'static str,
839    pub message: String,
840}
841
842/// Public history-budget information attached to the current workflow task.
843#[derive(Clone, Debug, Default, PartialEq, Eq)]
844pub struct WorkflowHistoryBudget {
845    pub event_count: u64,
846    pub size_bytes: Option<u64>,
847    pub continue_as_new_recommended: bool,
848    pub pressure: Option<String>,
849}
850
851#[doc(hidden)]
852#[derive(Clone, Debug)]
853pub struct ContinueAsNewRequest {
854    arguments: AvroValue,
855    options: ContinueAsNewOptions,
856}
857
858impl WorkflowCommandOptions {
859    pub fn new() -> Self {
860        Self::default()
861    }
862
863    pub fn reason(mut self, reason: impl Into<String>) -> Self {
864        self.reason = Some(reason.into());
865        self
866    }
867
868    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
869        self.request_id = Some(request_id.into());
870        self
871    }
872}
873
874/// The accepted, machine-readable result of a lifecycle command.
875#[derive(Clone, Debug, PartialEq)]
876pub struct WorkflowCommandResult {
877    pub command: WorkflowCommandKind,
878    pub workflow_id: String,
879    pub run_id: Option<String>,
880    pub outcome: Option<String>,
881    pub reason: Option<String>,
882    pub command_status: Option<String>,
883    pub raw: Value,
884}
885
886/// A new run continuing a failed run from its recorded activity boundary.
887#[derive(Clone, Debug, PartialEq)]
888pub struct WorkflowRedriveResult {
889    pub workflow_id: String,
890    pub source_run_id: String,
891    pub run_id: String,
892    pub resume_step_sequence: u64,
893    pub raw: Value,
894}
895
896/// A stable rejection returned by instance- or selected-run lifecycle commands.
897#[derive(Clone, Debug, Error)]
898#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
899pub struct WorkflowCommandRejection {
900    pub command: WorkflowCommandKind,
901    pub status: u16,
902    pub reason: String,
903    pub message: String,
904    pub workflow_id: String,
905    pub run_id: Option<String>,
906    pub target_scope: Option<String>,
907    pub body: Value,
908}
909
910/// Stable terminal categories returned by [`WorkflowHandle::result`].
911#[derive(Clone, Copy, Debug, PartialEq, Eq)]
912pub enum WorkflowTerminalKind {
913    Failed,
914    Cancelled,
915    Terminated,
916    TimedOut,
917}
918
919/// A typed terminal workflow outcome with durable identity and failure metadata.
920///
921/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
922/// of parsing its display representation. Fields remain `None` when an older
923/// server did not publish that metadata.
924#[derive(Clone, Debug, Error)]
925#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
926pub struct WorkflowTerminalOutcome {
927    pub kind: WorkflowTerminalKind,
928    pub workflow_id: String,
929    pub run_id: Option<String>,
930    pub reason: String,
931    pub failure_category: Option<String>,
932    pub failure_id: Option<String>,
933    pub exception_type: Option<String>,
934    pub exception_class: Option<String>,
935    pub non_retryable: Option<bool>,
936    pub message: Option<String>,
937    pub exception: Option<Value>,
938    pub raw: Value,
939}
940
941/// A worker-side activity settlement or heartbeat rejected by durable state.
942#[derive(Clone, Debug, Error)]
943#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
944pub struct ActivityTaskRejection {
945    pub operation: String,
946    pub status: u16,
947    pub reason: String,
948    pub task_id: String,
949    pub activity_attempt_id: String,
950    pub cancel_requested: bool,
951    pub can_continue: Option<bool>,
952    pub run_closed_reason: Option<String>,
953    pub body: Value,
954}
955
956/// Stable validation categories for [`ActivityOptions`].
957#[derive(Clone, Copy, Debug, PartialEq, Eq)]
958pub enum ActivityOptionsErrorKind {
959    EmptyTaskQueue,
960    EmptyRetryPolicy,
961    InvalidMaxAttempts,
962    BackoffWithoutRetryBudget,
963    TooManyBackoffIntervals,
964    InvalidBackoffCoefficient,
965    BackoffGenerationTooLarge,
966    BackoffOverflow,
967    EmptyNonRetryableErrorType,
968    TimeoutNotPositive,
969    TimeoutOverflow,
970    TimeoutOrder,
971}
972
973/// A machine-readable activity-options validation failure.
974#[derive(Clone, Debug, Error, PartialEq, Eq)]
975#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
976pub struct ActivityOptionsError {
977    pub kind: ActivityOptionsErrorKind,
978    pub field: Option<&'static str>,
979    pub message: String,
980}
981
982impl ActivityOptionsError {
983    fn new(
984        kind: ActivityOptionsErrorKind,
985        field: Option<&'static str>,
986        message: impl Into<String>,
987    ) -> Self {
988        Self {
989            kind,
990            field,
991            message: message.into(),
992        }
993    }
994}
995
996/// Stable terminal categories returned when an awaited activity does not succeed.
997#[derive(Clone, Copy, Debug, PartialEq, Eq)]
998pub enum ActivityFailureKind {
999    Failed,
1000    Cancelled,
1001    TimedOut,
1002}
1003
1004/// A stable, machine-readable terminal activity failure.
1005///
1006/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
1007/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
1008#[derive(Clone, Debug, Error, PartialEq)]
1009#[error("activity failed ({reason}): {message}")]
1010pub struct ActivityFailure {
1011    pub kind: ActivityFailureKind,
1012    pub reason: String,
1013    pub message: String,
1014    pub activity_execution_id: Option<String>,
1015    pub activity_attempt_id: Option<String>,
1016    pub activity_type: Option<String>,
1017    pub activity_class: Option<String>,
1018    pub attempt_number: Option<u64>,
1019    pub failure_id: Option<String>,
1020    pub failure_category: Option<String>,
1021    pub timeout_kind: Option<String>,
1022    pub non_retryable: bool,
1023    pub exception_type: Option<String>,
1024    pub exception_class: Option<String>,
1025    pub code: Option<Value>,
1026    pub exception: Option<Value>,
1027}
1028
1029/// Stable terminal categories returned when an awaited child does not succeed.
1030#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1031pub enum ChildWorkflowFailureKind {
1032    Failed,
1033    Cancelled,
1034    Terminated,
1035}
1036
1037/// A stable, machine-readable child workflow failure delivered to its parent.
1038///
1039/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
1040/// of parsing the display message. Child and parent identifiers retain the
1041/// relationship recorded in durable history across worker restarts.
1042#[derive(Clone, Debug, Error)]
1043#[error("child workflow failed ({reason}): {message}")]
1044pub struct ChildWorkflowFailure {
1045    pub kind: ChildWorkflowFailureKind,
1046    pub reason: String,
1047    pub message: String,
1048    pub parent_workflow_id: Option<String>,
1049    pub parent_workflow_run_id: Option<String>,
1050    pub child_workflow_id: Option<String>,
1051    pub child_workflow_run_id: Option<String>,
1052    pub child_workflow_type: Option<String>,
1053    pub failure_id: Option<String>,
1054    pub failure_category: Option<String>,
1055    pub exception_type: Option<String>,
1056    pub exception_class: Option<String>,
1057    pub non_retryable: bool,
1058    pub code: Option<Value>,
1059    pub exception: Option<Value>,
1060}
1061
1062/// The identity of one durable workflow execution.
1063#[derive(Clone, Debug, PartialEq, Eq)]
1064pub struct WorkflowIdentity {
1065    pub workflow_id: Option<String>,
1066    pub run_id: Option<String>,
1067}
1068
1069/// A successful child result together with its durable parent-child identity.
1070#[derive(Clone, Debug, PartialEq)]
1071pub struct ChildWorkflowResult {
1072    pub parent: WorkflowIdentity,
1073    pub child: WorkflowIdentity,
1074    pub child_workflow_type: Option<String>,
1075    pub result: Value,
1076}
1077
1078/// Lossless successful child result for fixed Avro Value workflows.
1079#[derive(Clone, Debug, PartialEq)]
1080pub struct ChildWorkflowAvroResult {
1081    pub parent: WorkflowIdentity,
1082    pub child: WorkflowIdentity,
1083    pub child_workflow_type: Option<String>,
1084    pub result: AvroValue,
1085}
1086
1087/// Stable user-facing identity for one member of a durable selection group.
1088#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1089#[serde(untagged)]
1090pub enum SelectionKey {
1091    Index(usize),
1092    Name(String),
1093}
1094
1095impl From<usize> for SelectionKey {
1096    fn from(value: usize) -> Self {
1097        Self::Index(value)
1098    }
1099}
1100
1101impl From<String> for SelectionKey {
1102    fn from(value: String) -> Self {
1103        Self::Name(value)
1104    }
1105}
1106
1107impl From<&str> for SelectionKey {
1108    fn from(value: &str) -> Self {
1109        Self::Name(value.to_string())
1110    }
1111}
1112
1113/// Typed result of explicitly awaiting a cancelled non-winning operation.
1114#[derive(Clone, Debug, Error, PartialEq, Eq)]
1115#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1116pub struct DurableOperationCancelled {
1117    pub selection_group_id: String,
1118    pub member_key: SelectionKey,
1119    pub member_index: usize,
1120    pub operation_kind: String,
1121    pub operation_identity: String,
1122}
1123
1124/// Stable identity for one enclosing deterministic parallel group.
1125///
1126/// The same fields are attached to every ordinary activity, timer, or child
1127/// workflow command in the group. Nested leaves carry an outer-to-inner path;
1128/// no Rust-specific wire command is introduced.
1129#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1130pub struct ParallelGroupMetadata {
1131    pub parallel_group_id: String,
1132    pub parallel_group_kind: String,
1133    pub parallel_group_base_sequence: u64,
1134    pub parallel_group_size: usize,
1135    pub parallel_group_index: usize,
1136    #[serde(default, skip_serializing_if = "Option::is_none")]
1137    pub parallel_group_mode: Option<String>,
1138    #[serde(default, skip_serializing_if = "Option::is_none")]
1139    pub selection_member_key: Option<SelectionKey>,
1140    #[serde(default, skip_serializing_if = "Option::is_none")]
1141    pub selection_member_index: Option<usize>,
1142    #[serde(default, skip_serializing_if = "Option::is_none")]
1143    pub selection_member_base_sequence: Option<u64>,
1144    #[serde(default, skip_serializing_if = "Option::is_none")]
1145    pub selection_member_size: Option<usize>,
1146    #[serde(default, skip_serializing_if = "Option::is_none")]
1147    pub selection_member_kind: Option<String>,
1148}
1149
1150/// One input-ordered result returned by [`WorkflowContext::parallel`].
1151#[derive(Clone, Debug, PartialEq)]
1152pub enum ParallelResult {
1153    Activity(Value),
1154    ChildWorkflow(ChildWorkflowResult),
1155    Timer,
1156    Signal(Vec<Value>),
1157    Condition(ConditionWaitResult),
1158    Group(Vec<ParallelResult>),
1159}
1160
1161/// Lossless fixed-Avro counterpart to [`ParallelResult`].
1162#[derive(Clone, Debug, PartialEq)]
1163pub enum ParallelAvroResult {
1164    Activity(AvroValue),
1165    ChildWorkflow(ChildWorkflowAvroResult),
1166    Timer,
1167    Signal(Vec<AvroValue>),
1168    Condition(ConditionWaitResult),
1169    Group(Vec<ParallelAvroResult>),
1170}
1171
1172impl ParallelAvroResult {
1173    fn into_json_result(self) -> Result<ParallelResult> {
1174        match self {
1175            Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1176            Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1177                parent: result.parent,
1178                child: result.child,
1179                child_workflow_type: result.child_workflow_type,
1180                result: result.result.into_json()?,
1181            })),
1182            Self::Timer => Ok(ParallelResult::Timer),
1183            Self::Signal(values) => Ok(ParallelResult::Signal(
1184                values
1185                    .into_iter()
1186                    .map(AvroValue::into_json)
1187                    .collect::<Result<Vec<_>>>()?,
1188            )),
1189            Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1190            Self::Group(results) => Ok(ParallelResult::Group(
1191                results
1192                    .into_iter()
1193                    .map(Self::into_json_result)
1194                    .collect::<Result<Vec<_>>>()?,
1195            )),
1196        }
1197    }
1198}
1199
1200/// One successful leaf retained when another parallel member failed.
1201#[derive(Clone, Debug, PartialEq)]
1202pub struct ParallelCompletion {
1203    pub member_path: Vec<usize>,
1204    pub result: ParallelResult,
1205}
1206
1207/// A deterministic join failed after some siblings had already completed.
1208///
1209/// `cause` retains the typed activity, child-workflow, cancellation, or codec
1210/// error. `completed` is declaration ordered and contains only durable
1211/// successes observed in the same replay. Late sibling completions can add
1212/// entries on a later replay without changing `member_path` or the selected
1213/// positional failure.
1214#[derive(Debug, Error)]
1215#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1216pub struct ParallelFailure {
1217    pub group_id: String,
1218    pub member_path: Vec<usize>,
1219    pub group_path: Vec<ParallelGroupMetadata>,
1220    pub completed: Vec<ParallelCompletion>,
1221    #[source]
1222    pub cause: Box<Error>,
1223}
1224
1225/// Stable validation error returned before an invalid group emits commands.
1226#[derive(Clone, Debug, Error, PartialEq, Eq)]
1227#[error("invalid deterministic parallel group ({reason}): {message}")]
1228pub struct ParallelGroupError {
1229    pub reason: &'static str,
1230    pub member_path: Vec<usize>,
1231    pub message: String,
1232}
1233
1234/// Cooperative workflow cancellation observed at an author-controlled point.
1235#[derive(Clone, Debug, Error, PartialEq, Eq)]
1236#[error("workflow cancellation was requested")]
1237pub struct WorkflowCancellationRequested;
1238
1239/// A forward saga failure followed by a terminal compensation failure.
1240#[derive(Debug, Error)]
1241#[error(
1242    "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1243)]
1244pub struct SagaCompensationFailure {
1245    pub initiating_failure: Box<Error>,
1246    pub compensation_failure: Box<Error>,
1247    pub compensation_activity_type: String,
1248    pub compensation_registration_order: usize,
1249}
1250
1251/// Server behavior when a parent closes while its child is still open.
1252#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1253pub enum ParentClosePolicy {
1254    #[default]
1255    Abandon,
1256    RequestCancel,
1257    Terminate,
1258}
1259
1260impl ParentClosePolicy {
1261    fn as_str(self) -> &'static str {
1262        match self {
1263            Self::Abandon => "abandon",
1264            Self::RequestCancel => "request_cancel",
1265            Self::Terminate => "terminate",
1266        }
1267    }
1268}
1269
1270/// Durable retry policy for one child workflow invocation.
1271#[derive(Clone, Debug, Default, PartialEq, Eq)]
1272pub struct ChildWorkflowRetryPolicy {
1273    pub max_attempts: Option<u32>,
1274    pub backoff_seconds: Vec<u64>,
1275    pub non_retryable_error_types: Vec<String>,
1276}
1277
1278/// Options recorded with a child-workflow command.
1279///
1280/// The task queue is mandatory so routing is explicit and replay-stable.
1281#[derive(Clone, Debug, PartialEq, Eq)]
1282pub struct ChildWorkflowOptions {
1283    pub task_queue: String,
1284    pub parent_close_policy: ParentClosePolicy,
1285    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1286    pub execution_timeout_seconds: Option<u64>,
1287    pub run_timeout_seconds: Option<u64>,
1288}
1289
1290impl ChildWorkflowOptions {
1291    pub fn new(task_queue: impl Into<String>) -> Self {
1292        Self {
1293            task_queue: task_queue.into(),
1294            parent_close_policy: ParentClosePolicy::Abandon,
1295            retry_policy: None,
1296            execution_timeout_seconds: None,
1297            run_timeout_seconds: None,
1298        }
1299    }
1300
1301    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1302        self.parent_close_policy = policy;
1303        self
1304    }
1305
1306    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1307        self.retry_policy = Some(policy);
1308        self
1309    }
1310
1311    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1312        self.execution_timeout_seconds = Some(seconds);
1313        self
1314    }
1315
1316    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1317        self.run_timeout_seconds = Some(seconds);
1318        self
1319    }
1320}
1321
1322/// Backoff intervals for one durable activity retry policy.
1323#[derive(Clone, Debug, PartialEq, Eq)]
1324pub enum ActivityBackoff {
1325    /// Use these intervals between attempts. The server repeats the final
1326    /// interval if the retry budget contains more attempts than entries.
1327    Explicit(Vec<Duration>),
1328    /// Generate one interval for every retry using integer exponential growth.
1329    Exponential {
1330        initial_interval: Duration,
1331        coefficient: u32,
1332        maximum_interval: Option<Duration>,
1333    },
1334}
1335
1336/// Durable server-side retry policy for one activity execution.
1337#[derive(Clone, Debug, Default, PartialEq, Eq)]
1338pub struct ActivityRetryPolicy {
1339    pub max_attempts: Option<u32>,
1340    pub backoff: Option<ActivityBackoff>,
1341    pub non_retryable_error_types: Vec<String>,
1342}
1343
1344impl ActivityRetryPolicy {
1345    /// Start a policy with a finite attempt budget, including the first attempt.
1346    pub fn new(max_attempts: u32) -> Self {
1347        Self {
1348            max_attempts: Some(max_attempts),
1349            ..Self::default()
1350        }
1351    }
1352
1353    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1354        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1355        self
1356    }
1357
1358    pub fn exponential_backoff(
1359        mut self,
1360        initial_interval: Duration,
1361        coefficient: u32,
1362        maximum_interval: Option<Duration>,
1363    ) -> Self {
1364        self.backoff = Some(ActivityBackoff::Exponential {
1365            initial_interval,
1366            coefficient,
1367            maximum_interval,
1368        });
1369        self
1370    }
1371
1372    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1373        self.non_retryable_error_types.push(error_type.into());
1374        self
1375    }
1376
1377    pub fn non_retryable_error_types(
1378        mut self,
1379        error_types: impl IntoIterator<Item = impl Into<String>>,
1380    ) -> Self {
1381        self.non_retryable_error_types
1382            .extend(error_types.into_iter().map(Into::into));
1383        self
1384    }
1385}
1386
1387/// Options recorded atomically on one deterministic `schedule_activity` command.
1388///
1389/// Durations are rounded up to whole seconds when encoded, so the server never
1390/// receives a shorter timeout or backoff than the caller requested.
1391#[derive(Clone, Debug, Default, PartialEq, Eq)]
1392pub struct ActivityOptions {
1393    pub task_queue: Option<String>,
1394    pub retry_policy: Option<ActivityRetryPolicy>,
1395    pub start_to_close_timeout: Option<Duration>,
1396    pub schedule_to_start_timeout: Option<Duration>,
1397    pub schedule_to_close_timeout: Option<Duration>,
1398    pub heartbeat_timeout: Option<Duration>,
1399}
1400
1401impl ActivityOptions {
1402    pub fn new() -> Self {
1403        Self::default()
1404    }
1405
1406    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1407        self.task_queue = Some(task_queue.into());
1408        self
1409    }
1410
1411    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1412        self.retry_policy = Some(policy);
1413        self
1414    }
1415
1416    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1417        self.start_to_close_timeout = Some(timeout);
1418        self
1419    }
1420
1421    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1422        self.schedule_to_start_timeout = Some(timeout);
1423        self
1424    }
1425
1426    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1427        self.schedule_to_close_timeout = Some(timeout);
1428        self
1429    }
1430
1431    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1432        self.heartbeat_timeout = Some(timeout);
1433        self
1434    }
1435
1436    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1437        if self
1438            .task_queue
1439            .as_deref()
1440            .is_some_and(|queue| queue.trim().is_empty())
1441        {
1442            return Err(ActivityOptionsError::new(
1443                ActivityOptionsErrorKind::EmptyTaskQueue,
1444                Some("task_queue"),
1445                "task_queue must not be empty",
1446            ));
1447        }
1448
1449        for (field, value) in [
1450            ("start_to_close_timeout", self.start_to_close_timeout),
1451            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1452            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1453            ("heartbeat_timeout", self.heartbeat_timeout),
1454        ] {
1455            if value.is_some_and(|value| value.is_zero()) {
1456                return Err(ActivityOptionsError::new(
1457                    ActivityOptionsErrorKind::TimeoutNotPositive,
1458                    Some(field),
1459                    format!("{field} must be positive"),
1460                ));
1461            }
1462        }
1463
1464        validate_timeout_order(
1465            "heartbeat_timeout",
1466            self.heartbeat_timeout,
1467            "start_to_close_timeout",
1468            self.start_to_close_timeout,
1469        )?;
1470        validate_timeout_order(
1471            "start_to_close_timeout",
1472            self.start_to_close_timeout,
1473            "schedule_to_close_timeout",
1474            self.schedule_to_close_timeout,
1475        )?;
1476        validate_timeout_order(
1477            "schedule_to_start_timeout",
1478            self.schedule_to_start_timeout,
1479            "schedule_to_close_timeout",
1480            self.schedule_to_close_timeout,
1481        )?;
1482
1483        Ok(ValidatedActivityOptions {
1484            task_queue: self.task_queue.clone(),
1485            retry_policy: self
1486                .retry_policy
1487                .as_ref()
1488                .map(validate_activity_retry_policy)
1489                .transpose()?,
1490            start_to_close_timeout: timeout_seconds(
1491                "start_to_close_timeout",
1492                self.start_to_close_timeout,
1493            )?,
1494            schedule_to_start_timeout: timeout_seconds(
1495                "schedule_to_start_timeout",
1496                self.schedule_to_start_timeout,
1497            )?,
1498            schedule_to_close_timeout: timeout_seconds(
1499                "schedule_to_close_timeout",
1500                self.schedule_to_close_timeout,
1501            )?,
1502            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1503        })
1504    }
1505}
1506
1507/// A deferred durable leaf or nested group for [`WorkflowContext::parallel`].
1508///
1509/// Constructors capture arguments but perform no I/O. The join validates the
1510/// complete tree, attaches the existing parallel-group metadata to every
1511/// ordinary command, schedules all leaves, and then suspends.
1512pub enum ParallelOperation {
1513    Activity {
1514        activity_type: String,
1515        options: ActivityOptions,
1516        arguments: Result<AvroValue>,
1517    },
1518    ChildWorkflow {
1519        workflow_type: String,
1520        options: ChildWorkflowOptions,
1521        arguments: Result<AvroValue>,
1522    },
1523    Timer(Duration),
1524    Signal(String),
1525    Condition {
1526        options: ConditionWaitOptions,
1527        predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1528    },
1529    Group(Vec<ParallelOperation>),
1530}
1531
1532impl ParallelOperation {
1533    pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1534        Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1535    }
1536
1537    pub fn activity_with_options<T: Serialize>(
1538        activity_type: impl Into<String>,
1539        options: ActivityOptions,
1540        args: T,
1541    ) -> Self {
1542        Self::Activity {
1543            activity_type: activity_type.into(),
1544            options,
1545            arguments: AvroValue::from_serialize(&args),
1546        }
1547    }
1548
1549    pub fn child_workflow<T: Serialize>(
1550        workflow_type: impl Into<String>,
1551        options: ChildWorkflowOptions,
1552        args: T,
1553    ) -> Self {
1554        Self::ChildWorkflow {
1555            workflow_type: workflow_type.into(),
1556            options,
1557            arguments: AvroValue::from_serialize(&args),
1558        }
1559    }
1560
1561    pub fn timer(duration: Duration) -> Self {
1562        Self::Timer(duration)
1563    }
1564
1565    pub fn signal(signal_name: impl Into<String>) -> Self {
1566        Self::Signal(signal_name.into())
1567    }
1568
1569    pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1570    where
1571        F: Fn() -> Result<bool> + Send + 'static,
1572    {
1573        Self::Condition {
1574            options,
1575            predicate: Box::new(predicate),
1576        }
1577    }
1578
1579    pub fn group(operations: Vec<ParallelOperation>) -> Self {
1580        Self::Group(operations)
1581    }
1582}
1583
1584#[derive(Clone, Debug)]
1585struct ValidatedActivityOptions {
1586    task_queue: Option<String>,
1587    retry_policy: Option<Value>,
1588    start_to_close_timeout: Option<u64>,
1589    schedule_to_start_timeout: Option<u64>,
1590    schedule_to_close_timeout: Option<u64>,
1591    heartbeat_timeout: Option<u64>,
1592}
1593
1594fn validate_timeout_order(
1595    smaller_name: &'static str,
1596    smaller: Option<Duration>,
1597    larger_name: &'static str,
1598    larger: Option<Duration>,
1599) -> std::result::Result<(), ActivityOptionsError> {
1600    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1601        return Err(ActivityOptionsError::new(
1602            ActivityOptionsErrorKind::TimeoutOrder,
1603            Some(smaller_name),
1604            format!("{smaller_name} must be <= {larger_name}"),
1605        ));
1606    }
1607    Ok(())
1608}
1609
1610fn timeout_seconds(
1611    field: &'static str,
1612    value: Option<Duration>,
1613) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1614    value
1615        .map(|value| {
1616            activity_protocol_seconds(value).ok_or_else(|| {
1617                ActivityOptionsError::new(
1618                    ActivityOptionsErrorKind::TimeoutOverflow,
1619                    Some(field),
1620                    format!("{field} is too large for the worker protocol"),
1621                )
1622            })
1623        })
1624        .transpose()
1625}
1626
1627fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1628    value
1629        .as_secs()
1630        .checked_add(u64::from(value.subsec_nanos() > 0))
1631}
1632
1633fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1634    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1635}
1636
1637fn validate_activity_retry_policy(
1638    policy: &ActivityRetryPolicy,
1639) -> std::result::Result<Value, ActivityOptionsError> {
1640    if policy.max_attempts.is_none()
1641        && policy.backoff.is_none()
1642        && policy.non_retryable_error_types.is_empty()
1643    {
1644        return Err(ActivityOptionsError::new(
1645            ActivityOptionsErrorKind::EmptyRetryPolicy,
1646            Some("retry_policy"),
1647            "retry_policy must configure at least one field",
1648        ));
1649    }
1650    if policy.max_attempts == Some(0) {
1651        return Err(ActivityOptionsError::new(
1652            ActivityOptionsErrorKind::InvalidMaxAttempts,
1653            Some("retry_policy.max_attempts"),
1654            "max_attempts must be >= 1",
1655        ));
1656    }
1657    if policy
1658        .non_retryable_error_types
1659        .iter()
1660        .any(|error_type| error_type.trim().is_empty())
1661    {
1662        return Err(ActivityOptionsError::new(
1663            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1664            Some("retry_policy.non_retryable_error_types"),
1665            "non_retryable_error_types must not contain empty values",
1666        ));
1667    }
1668
1669    let backoff_seconds = match &policy.backoff {
1670        None => None,
1671        Some(backoff) => {
1672            let max_attempts = policy.max_attempts.ok_or_else(|| {
1673                ActivityOptionsError::new(
1674                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1675                    Some("retry_policy.backoff"),
1676                    "backoff requires max_attempts",
1677                )
1678            })?;
1679            let retry_count = max_attempts.saturating_sub(1) as usize;
1680            let intervals = match backoff {
1681                ActivityBackoff::Explicit(intervals) => {
1682                    if intervals.len() > retry_count {
1683                        return Err(ActivityOptionsError::new(
1684                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
1685                            Some("retry_policy.backoff"),
1686                            "backoff interval count must not exceed max_attempts - 1",
1687                        ));
1688                    }
1689                    intervals.clone()
1690                }
1691                ActivityBackoff::Exponential {
1692                    initial_interval,
1693                    coefficient,
1694                    maximum_interval,
1695                } => {
1696                    if *coefficient < 1 {
1697                        return Err(ActivityOptionsError::new(
1698                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1699                            Some("retry_policy.backoff.coefficient"),
1700                            "backoff coefficient must be >= 1",
1701                        ));
1702                    }
1703                    if retry_count > 10_000 {
1704                        return Err(ActivityOptionsError::new(
1705                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1706                            Some("retry_policy.max_attempts"),
1707                            "generated backoff supports at most 10000 retry intervals",
1708                        ));
1709                    }
1710                    let mut current = *initial_interval;
1711                    let mut intervals = Vec::with_capacity(retry_count);
1712                    for _ in 0..retry_count {
1713                        let interval = maximum_interval
1714                            .map(|maximum| current.min(maximum))
1715                            .unwrap_or(current);
1716                        intervals.push(interval);
1717                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
1718                            break;
1719                        }
1720                        current = current.checked_mul(*coefficient).ok_or_else(|| {
1721                            ActivityOptionsError::new(
1722                                ActivityOptionsErrorKind::BackoffOverflow,
1723                                Some("retry_policy.backoff"),
1724                                "generated backoff interval overflowed",
1725                            )
1726                        })?;
1727                    }
1728                    intervals
1729                }
1730            };
1731            Some(
1732                intervals
1733                    .into_iter()
1734                    .map(|interval| {
1735                        activity_protocol_seconds(interval).ok_or_else(|| {
1736                            ActivityOptionsError::new(
1737                                ActivityOptionsErrorKind::BackoffOverflow,
1738                                Some("retry_policy.backoff"),
1739                                "backoff interval is too large for the worker protocol",
1740                            )
1741                        })
1742                    })
1743                    .collect::<std::result::Result<Vec<_>, _>>()?,
1744            )
1745        }
1746    };
1747
1748    let mut encoded = serde_json::Map::new();
1749    if let Some(max_attempts) = policy.max_attempts {
1750        encoded.insert("max_attempts".to_string(), json!(max_attempts));
1751    }
1752    if let Some(backoff_seconds) = backoff_seconds {
1753        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1754    }
1755    if !policy.non_retryable_error_types.is_empty() {
1756        let mut canonical_error_types = Vec::new();
1757        for error_type in policy
1758            .non_retryable_error_types
1759            .iter()
1760            .map(|error_type| error_type.trim())
1761        {
1762            if !canonical_error_types.contains(&error_type) {
1763                canonical_error_types.push(error_type);
1764            }
1765        }
1766        encoded.insert(
1767            "non_retryable_error_types".to_string(),
1768            json!(canonical_error_types),
1769        );
1770    }
1771    Ok(Value::Object(encoded))
1772}
1773
1774/// A stable, machine-readable failure raised when workflow code no longer
1775/// reconstructs the durable command stream recorded in history.
1776#[derive(Clone, Debug, Error)]
1777#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1778pub struct ReplayFailure {
1779    pub reason: String,
1780    pub sequence: Option<u64>,
1781    pub expected: Option<String>,
1782    pub actual: Option<String>,
1783    pub message: String,
1784}
1785
1786impl ReplayFailure {
1787    fn new(
1788        reason: impl Into<String>,
1789        sequence: Option<u64>,
1790        expected: Option<String>,
1791        actual: Option<String>,
1792        message: impl Into<String>,
1793    ) -> Self {
1794        Self {
1795            reason: reason.into(),
1796            sequence,
1797            expected,
1798            actual,
1799            message: message.into(),
1800        }
1801    }
1802}
1803
1804/// A stable, machine-readable workflow query or query-task settlement failure.
1805#[derive(Clone, Debug, Error)]
1806#[error("query failed ({reason}, HTTP {status}): {message}")]
1807pub struct QueryFailure {
1808    pub status: u16,
1809    pub reason: String,
1810    pub message: String,
1811    pub body: Value,
1812}
1813
1814/// A stable failure returned when a server rejects an SDK protocol version.
1815#[derive(Clone, Debug, Error)]
1816#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1817pub struct ProtocolFailure {
1818    pub status: u16,
1819    pub reason: String,
1820    pub message: String,
1821    pub supported_version: Option<String>,
1822    pub requested_version: Option<String>,
1823    pub body: Value,
1824}
1825
1826#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1827pub struct PayloadEnvelope {
1828    pub codec: String,
1829    pub blob: String,
1830}
1831
1832impl PayloadEnvelope {
1833    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1834        encode_payload(value, DEFAULT_CODEC)
1835    }
1836
1837    /// Encode an explicit typed value, including the bytes branch that JSON
1838    /// serialization cannot represent.
1839    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1840        encode_avro_value(value)
1841    }
1842}
1843
1844/// Native adapter for the fixed language-neutral Avro Value schema.
1845#[derive(Clone, Debug)]
1846pub enum AvroValue {
1847    Null,
1848    Boolean(bool),
1849    Long(i64),
1850    Double(f64),
1851    Bytes(Vec<u8>),
1852    String(String),
1853    Array(Vec<AvroValue>),
1854    Map(BTreeMap<String, AvroValue>),
1855}
1856
1857impl PartialEq for AvroValue {
1858    fn eq(&self, other: &Self) -> bool {
1859        match (self, other) {
1860            (Self::Null, Self::Null) => true,
1861            (Self::Boolean(left), Self::Boolean(right)) => left == right,
1862            (Self::Long(left), Self::Long(right)) => left == right,
1863            (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1864            (Self::Bytes(left), Self::Bytes(right)) => left == right,
1865            (Self::String(left), Self::String(right)) => left == right,
1866            (Self::Array(left), Self::Array(right)) => left == right,
1867            (Self::Map(left), Self::Map(right)) => left == right,
1868            _ => false,
1869        }
1870    }
1871}
1872
1873impl AvroValue {
1874    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1875        Self::from_serde_value(
1876            serde_value::to_value(value).map_err(|error| {
1877                Error::Codec(format!("could not adapt value for Avro: {error}"))
1878            })?,
1879        )
1880    }
1881
1882    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1883        use serde_value::Value as SerdeValue;
1884
1885        match value {
1886            SerdeValue::Unit => Ok(Self::Null),
1887            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1888            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1889            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1890            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1891            SerdeValue::I64(value) => Ok(Self::Long(value)),
1892            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1893            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1894            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1895            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1896                Error::Codec(
1897                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1898                        .to_string(),
1899                )
1900            }),
1901            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1902            SerdeValue::F64(value) => Self::finite_double(value),
1903            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1904            SerdeValue::String(value) => Ok(Self::String(value)),
1905            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1906            SerdeValue::Option(None) => Ok(Self::Null),
1907            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1908                Self::from_serde_value(*value)
1909            }
1910            SerdeValue::Seq(values) => values
1911                .into_iter()
1912                .map(Self::from_serde_value)
1913                .collect::<Result<Vec<_>>>()
1914                .map(Self::Array),
1915            SerdeValue::Map(values) => values
1916                .into_iter()
1917                .map(|(key, value)| {
1918                    let SerdeValue::String(key) = key else {
1919                        return Err(Error::Codec(
1920                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1921                        ));
1922                    };
1923
1924                    Ok((key, Self::from_serde_value(value)?))
1925                })
1926                .collect::<Result<BTreeMap<_, _>>>()
1927                .map(Self::Map),
1928        }
1929    }
1930
1931    fn finite_double(value: f64) -> Result<Self> {
1932        if !value.is_finite() {
1933            return Err(Error::Codec(
1934                "non_finite_float: Avro Value doubles must be finite".to_string(),
1935            ));
1936        }
1937
1938        Ok(Self::Double(value))
1939    }
1940
1941    fn into_json(self) -> Result<Value> {
1942        match self {
1943            Self::Null => Ok(Value::Null),
1944            Self::Boolean(value) => Ok(Value::Bool(value)),
1945            Self::Long(value) => Ok(Value::Number(value.into())),
1946            Self::Double(value) => serde_json::Number::from_f64(value)
1947                .map(Value::Number)
1948                .ok_or_else(|| {
1949                    Error::Codec(
1950                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1951                    )
1952                }),
1953            Self::Bytes(value) => Ok(json!({
1954                "$type": "bytes",
1955                "base64": BASE64.encode(value),
1956            })),
1957            Self::String(value) => Ok(Value::String(value)),
1958            Self::Array(values) => values
1959                .into_iter()
1960                .map(Self::into_json)
1961                .collect::<Result<Vec<_>>>()
1962                .map(Value::Array),
1963            Self::Map(values) => values
1964                .into_iter()
1965                .map(|(key, value)| Ok((key, value.into_json()?)))
1966                .collect::<Result<serde_json::Map<_, _>>>()
1967                .map(Value::Object),
1968        }
1969    }
1970
1971    fn into_serde_value(self) -> serde_value::Value {
1972        use serde_value::Value as SerdeValue;
1973
1974        match self {
1975            Self::Null => SerdeValue::Unit,
1976            Self::Boolean(value) => SerdeValue::Bool(value),
1977            Self::Long(value) => SerdeValue::I64(value),
1978            Self::Double(value) => SerdeValue::F64(value),
1979            Self::Bytes(value) => SerdeValue::Bytes(value),
1980            Self::String(value) => SerdeValue::String(value),
1981            Self::Array(values) => {
1982                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1983            }
1984            Self::Map(values) => SerdeValue::Map(
1985                values
1986                    .into_iter()
1987                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1988                    .collect(),
1989            ),
1990        }
1991    }
1992
1993    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1994        self.into_serde_value().deserialize_into().map_err(|error| {
1995            Error::Codec(format!(
1996                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1997            ))
1998        })
1999    }
2000}
2001
2002impl Serialize for AvroValue {
2003    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
2004    where
2005        S: Serializer,
2006    {
2007        match self {
2008            Self::Null => serializer.serialize_unit(),
2009            Self::Boolean(value) => serializer.serialize_bool(*value),
2010            Self::Long(value) => serializer.serialize_i64(*value),
2011            Self::Double(value) => serializer.serialize_f64(*value),
2012            Self::Bytes(value) => serializer.serialize_bytes(value),
2013            Self::String(value) => serializer.serialize_str(value),
2014            Self::Array(values) => {
2015                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2016                for value in values {
2017                    sequence.serialize_element(value)?;
2018                }
2019                sequence.end()
2020            }
2021            Self::Map(values) => {
2022                let mut map = serializer.serialize_map(Some(values.len()))?;
2023                for (key, value) in values {
2024                    map.serialize_entry(key, value)?;
2025                }
2026                map.end()
2027            }
2028        }
2029    }
2030}
2031
2032pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2033    let datum = avro_value_to_datum(value)?;
2034    let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2035        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2036    let mut bytes = Vec::with_capacity(datum.len() + 10);
2037    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2038    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2039    bytes.extend_from_slice(&datum);
2040    Ok(PayloadEnvelope {
2041        codec: DEFAULT_CODEC.to_string(),
2042        blob: BASE64.encode(bytes),
2043    })
2044}
2045
2046pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2047    if envelope.codec != DEFAULT_CODEC {
2048        return Err(unsupported_payload_codec(&envelope.codec));
2049    }
2050    decode_avro_value_blob(&envelope.blob)
2051}
2052
2053pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2054    let blob = match codec {
2055        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2056        other => return Err(unsupported_payload_codec(other)),
2057    };
2058
2059    Ok(PayloadEnvelope {
2060        codec: codec.to_string(),
2061        blob,
2062    })
2063}
2064
2065pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2066    match envelope.codec.as_str() {
2067        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2068        other => Err(unsupported_payload_codec(other)),
2069    }
2070}
2071
2072fn handler_type_error<T>(
2073    handler_kind: HandlerKind,
2074    handler_name: &str,
2075    value_kind: HandlerValueKind,
2076    message: impl Into<String>,
2077) -> Error {
2078    Error::HandlerType {
2079        handler_kind,
2080        handler_name: handler_name.to_string(),
2081        value_kind,
2082        rust_type: type_name::<T>(),
2083        message: message.into(),
2084    }
2085}
2086
2087fn decode_handler_input<T: DeserializeOwned>(
2088    arguments: AvroValue,
2089    handler_kind: HandlerKind,
2090    handler_name: &str,
2091) -> Result<T> {
2092    let argument = match arguments {
2093        AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2094            arguments.pop().expect("one typed handler argument")
2095        }
2096        AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2097        AvroValue::Array(arguments) => {
2098            return Err(handler_type_error::<T>(
2099                handler_kind,
2100                handler_name,
2101                HandlerValueKind::Input,
2102                format!(
2103                    "typed handlers accept one request value, but the task carried {} arguments",
2104                    arguments.len()
2105                ),
2106            ));
2107        }
2108        argument => argument,
2109    };
2110
2111    argument.deserialize().map_err(|error| {
2112        handler_type_error::<T>(
2113            handler_kind,
2114            handler_name,
2115            HandlerValueKind::Input,
2116            error.to_string(),
2117        )
2118    })
2119}
2120
2121fn encode_handler_result<T: Serialize>(
2122    result: &T,
2123    handler_kind: HandlerKind,
2124    handler_name: &str,
2125) -> Result<AvroValue> {
2126    AvroValue::from_serialize(result).map_err(|error| {
2127        handler_type_error::<T>(
2128            handler_kind,
2129            handler_name,
2130            HandlerValueKind::Result,
2131            error.to_string(),
2132        )
2133    })
2134}
2135
2136fn decode_handler_result<T: DeserializeOwned>(
2137    result: AvroValue,
2138    handler_kind: HandlerKind,
2139    handler_name: &str,
2140) -> Result<T> {
2141    result.deserialize().map_err(|error| {
2142        handler_type_error::<T>(
2143            handler_kind,
2144            handler_name,
2145            HandlerValueKind::Result,
2146            error.to_string(),
2147        )
2148    })
2149}
2150
2151#[cfg(test)]
2152fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2153    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2154}
2155
2156fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2157    validate_payload_codec(fallback_codec)?;
2158
2159    if value.is_null() {
2160        return Ok(Value::Null);
2161    }
2162
2163    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2164        return decode_blob(blob, codec);
2165    }
2166
2167    if let Some(blob) = value.as_str() {
2168        return decode_blob(blob, fallback_codec);
2169    }
2170
2171    Err(untagged_payload_value())
2172}
2173
2174fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2175    let envelope = match codec {
2176        DEFAULT_CODEC => encode_avro_value(value)?,
2177        other => return Err(unsupported_payload_codec(other)),
2178    };
2179    Ok(serde_json::to_value(envelope)?)
2180}
2181
2182fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2183    validate_payload_codec(fallback_codec)?;
2184
2185    if value.is_null() {
2186        return Ok(AvroValue::Null);
2187    }
2188
2189    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2190        validate_payload_codec(codec)?;
2191        return decode_avro_value_blob(blob);
2192    }
2193
2194    if let Some(blob) = value.as_str() {
2195        return match fallback_codec {
2196            DEFAULT_CODEC => decode_avro_value_blob(blob),
2197            other => Err(unsupported_payload_codec(other)),
2198        };
2199    }
2200
2201    Err(untagged_payload_value())
2202}
2203
2204fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2205    match value {
2206        AvroValue::Null => AvroValue::Array(Vec::new()),
2207        AvroValue::Array(_) => value,
2208        other => AvroValue::Array(vec![other]),
2209    }
2210}
2211
2212fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2213    match codec {
2214        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2215        other => Err(unsupported_payload_codec(other)),
2216    }
2217}
2218
2219fn validate_payload_codec(codec: &str) -> Result<()> {
2220    match codec {
2221        DEFAULT_CODEC => Ok(()),
2222        MISSING_TASK_PAYLOAD_CODEC => {
2223            Err(invalid_task_payload_codec("task payload_codec is missing"))
2224        }
2225        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2226        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2227            "task payload_codec must be a string",
2228        )),
2229        other => Err(unsupported_payload_codec(other)),
2230    }
2231}
2232
2233fn invalid_task_payload_codec(reason: &str) -> Error {
2234    Error::Codec(format!(
2235        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2236    ))
2237}
2238
2239fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2240    let Some(object) = value.as_object() else {
2241        return Ok(None);
2242    };
2243    if !object.contains_key("codec") && !object.contains_key("blob") {
2244        return Ok(None);
2245    }
2246
2247    let codec = object
2248        .get("codec")
2249        .and_then(Value::as_str)
2250        .ok_or_else(invalid_payload_envelope)?;
2251    validate_payload_codec(codec)?;
2252    let blob = object
2253        .get("blob")
2254        .and_then(Value::as_str)
2255        .ok_or_else(invalid_payload_envelope)?;
2256    Ok(Some((codec, blob)))
2257}
2258
2259fn invalid_payload_envelope() -> Error {
2260    Error::Codec(
2261        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2262            .to_string(),
2263    )
2264}
2265
2266fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2267    for command in commands {
2268        let Some(command) = command.as_object() else {
2269            continue;
2270        };
2271        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2272            continue;
2273        };
2274        let Some(payload_field) = workflow_command_payload_field(command_type) else {
2275            continue;
2276        };
2277
2278        if let Some(codec) = command.get("payload_codec") {
2279            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2280            validate_payload_codec(codec)?;
2281        }
2282
2283        let payload = command
2284            .get(payload_field)
2285            .ok_or_else(invalid_payload_envelope)?;
2286        if runtime_payloads::Reference::parse(payload)?.is_none() {
2287            validate_outbound_payload_envelope(payload)?;
2288        }
2289    }
2290    Ok(())
2291}
2292
2293fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2294    if commands.iter().any(|command| {
2295        command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2296            && command
2297                .get("condition_wait_occurrence_id")
2298                .and_then(Value::as_str)
2299                .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2300    }) {
2301        CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2302    } else if commands.iter().any(|command| {
2303        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2304            && command.get("attribute_types").is_some()
2305    }) {
2306        TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2307    } else if commands
2308        .iter()
2309        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2310    {
2311        MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2312    } else if commands
2313        .iter()
2314        .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2315    {
2316        CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2317    } else if commands.iter().any(|command| {
2318        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2319    }) {
2320        SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2321    } else {
2322        WORKER_PROTOCOL_VERSION
2323    }
2324}
2325
2326fn workflow_completion_protocol_version_with_message_streams(
2327    commands: &[Value],
2328    has_message_stream_metadata: bool,
2329) -> &'static str {
2330    let command_protocol = workflow_completion_protocol_version(commands);
2331    if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2332        MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2333    } else {
2334        command_protocol
2335    }
2336}
2337
2338fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2339    match command_type {
2340        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2341        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2342        "start_service_operation" => Some("request_payload"),
2343        "upsert_memo" => Some("entries"),
2344        _ => None,
2345    }
2346}
2347
2348fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2349    let Some((codec, blob)) = payload_envelope_parts(value)? else {
2350        return Err(untagged_payload_value());
2351    };
2352    validate_payload_codec(codec)?;
2353    decode_avro_value_blob(blob)?;
2354    Ok(())
2355}
2356
2357fn unsupported_payload_codec(codec: &str) -> Error {
2358    Error::Codec(format!(
2359        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2360    ))
2361}
2362
2363fn untagged_payload_value() -> Error {
2364    Error::Codec(
2365        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2366            .to_string(),
2367    )
2368}
2369
2370fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2371    let bytes = BASE64.decode(blob).map_err(|err| {
2372        Error::Codec(format!(
2373            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2374        ))
2375    })?;
2376
2377    if serde_json::from_slice::<Value>(&bytes).is_ok() {
2378        return Err(unsupported_payload_codec("json"));
2379    }
2380
2381    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2382        return Err(Error::Codec(
2383            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2384        ));
2385    }
2386
2387    let fingerprint: [u8; 8] = bytes[2..10]
2388        .try_into()
2389        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2390    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2391        return Err(Error::Codec(format!(
2392            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2393            fingerprint
2394                .iter()
2395                .map(|byte| format!("{byte:02x}"))
2396                .collect::<String>()
2397        )));
2398    }
2399
2400    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2401    // The current fingerprint selects the current immutable schema, so reader
2402    // resolution would only re-walk the same recursive union. Future retained
2403    // writer fingerprints supply a distinct reader schema in this branch.
2404    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2405    if datum_reader.truncated {
2406        return Err(Error::Codec(
2407            "invalid_payload_framing: truncated Avro Value datum".to_string(),
2408        ));
2409    }
2410    let datum = datum.map_err(|err| {
2411        Error::Codec(format!(
2412            "invalid_payload_framing: malformed Avro Value datum: {err}"
2413        ))
2414    })?;
2415    if datum_reader.remaining() != 0 {
2416        return Err(Error::Codec(format!(
2417            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2418            datum_reader.remaining()
2419        )));
2420    }
2421    avro_value_from_datum(datum)
2422}
2423
2424struct StrictAvroDatumReader<'a> {
2425    bytes: &'a [u8],
2426    offset: usize,
2427    truncated: bool,
2428}
2429
2430impl<'a> StrictAvroDatumReader<'a> {
2431    fn new(bytes: &'a [u8]) -> Self {
2432        Self {
2433            bytes,
2434            offset: 0,
2435            truncated: false,
2436        }
2437    }
2438
2439    fn remaining(&self) -> usize {
2440        self.bytes.len() - self.offset
2441    }
2442}
2443
2444impl Read for StrictAvroDatumReader<'_> {
2445    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2446        let count = buffer.len().min(self.remaining());
2447        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2448        self.offset += count;
2449        if count < buffer.len() {
2450            self.truncated = true;
2451        }
2452
2453        Ok(count)
2454    }
2455}
2456
2457fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2458    let branch = match value {
2459        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2460        AvroValue::Boolean(value) => AvroDatum::Union(
2461            1,
2462            Box::new(AvroDatum::Record(vec![(
2463                "boolean".to_string(),
2464                AvroDatum::Boolean(*value),
2465            )])),
2466        ),
2467        AvroValue::Long(value) => AvroDatum::Union(
2468            2,
2469            Box::new(AvroDatum::Record(vec![(
2470                "long".to_string(),
2471                AvroDatum::Long(*value),
2472            )])),
2473        ),
2474        AvroValue::Double(value) => {
2475            if !value.is_finite() {
2476                return Err(Error::Codec(
2477                    "non_finite_float: Avro Value doubles must be finite".to_string(),
2478                ));
2479            }
2480            AvroDatum::Union(
2481                3,
2482                Box::new(AvroDatum::Record(vec![(
2483                    "double".to_string(),
2484                    AvroDatum::Double(*value),
2485                )])),
2486            )
2487        }
2488        AvroValue::Bytes(value) => AvroDatum::Union(
2489            4,
2490            Box::new(AvroDatum::Record(vec![(
2491                "bytes".to_string(),
2492                AvroDatum::Bytes(value.clone()),
2493            )])),
2494        ),
2495        AvroValue::String(value) => AvroDatum::Union(
2496            5,
2497            Box::new(AvroDatum::Record(vec![(
2498                "string".to_string(),
2499                AvroDatum::String(value.clone()),
2500            )])),
2501        ),
2502        AvroValue::Array(values) => AvroDatum::Union(
2503            6,
2504            Box::new(AvroDatum::Record(vec![(
2505                "items".to_string(),
2506                AvroDatum::Array(
2507                    values
2508                        .iter()
2509                        .map(avro_value_to_datum)
2510                        .collect::<Result<Vec<_>>>()?,
2511                ),
2512            )])),
2513        ),
2514        AvroValue::Map(values) => AvroDatum::Union(
2515            7,
2516            Box::new(AvroDatum::Record(vec![(
2517                "entries".to_string(),
2518                AvroDatum::Array(
2519                    values
2520                        .iter()
2521                        .map(|(key, value)| {
2522                            Ok(AvroDatum::Record(vec![
2523                                ("key".to_string(), AvroDatum::String(key.clone())),
2524                                ("value".to_string(), avro_value_to_datum(value)?),
2525                            ]))
2526                        })
2527                        .collect::<Result<Vec<_>>>()?,
2528                ),
2529            )])),
2530        ),
2531    };
2532    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2533}
2534
2535fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2536    let AvroDatum::Record(mut outer) = datum else {
2537        return Err(Error::Codec(
2538            "invalid_payload_framing: datum is not a Value record".to_string(),
2539        ));
2540    };
2541    let (_, branch) = outer
2542        .pop()
2543        .filter(|(name, _)| name == "value")
2544        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2545    let AvroDatum::Union(_, branch) = branch else {
2546        return Err(Error::Codec(
2547            "invalid_payload_framing: invalid Value union".to_string(),
2548        ));
2549    };
2550    match *branch {
2551        AvroDatum::Null => Ok(AvroValue::Null),
2552        AvroDatum::Record(mut fields) => {
2553            let (name, value) = fields.pop().ok_or_else(|| {
2554                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2555            })?;
2556            match (name.as_str(), value) {
2557                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2558                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2559                ("double", AvroDatum::Double(value)) if value.is_finite() => {
2560                    Ok(AvroValue::Double(value))
2561                }
2562                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2563                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2564                ("items", AvroDatum::Array(values)) => values
2565                    .into_iter()
2566                    .map(avro_value_from_datum)
2567                    .collect::<Result<Vec<_>>>()
2568                    .map(AvroValue::Array),
2569                ("entries", AvroDatum::Map(values)) => values
2570                    .into_iter()
2571                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2572                    .collect::<Result<BTreeMap<_, _>>>()
2573                    .map(AvroValue::Map),
2574                _ => Err(Error::Codec(
2575                    "invalid_payload_framing: unknown Value branch".to_string(),
2576                )),
2577            }
2578        }
2579        _ => Err(Error::Codec(
2580            "invalid_payload_framing: invalid Value branch".to_string(),
2581        )),
2582    }
2583}
2584
2585fn avro_value_schema() -> Result<&'static Schema> {
2586    match AVRO_VALUE_SCHEMA.get_or_init(|| {
2587        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2588            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2589    }) {
2590        Ok(schema) => Ok(schema),
2591        Err(message) => Err(Error::Codec(message.clone())),
2592    }
2593}
2594
2595fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2596    match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2597        // Apache Avro's Value::Map uses a randomized HashMap. Arrays and maps
2598        // have the same block representation when each ordered array record
2599        // contains the map key followed by its value, so this encoding-only
2600        // adaptation lets the official encoder retain BTreeMap wire order.
2601        let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2602            .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2603        let entries_schema = schema
2604            .pointer_mut("/fields/0/type/7/fields/0/type")
2605            .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2606        if *entries_schema != json!({"type": "map", "values": "Value"}) {
2607            return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2608        }
2609        *entries_schema = json!({
2610            "type": "array",
2611            "items": {
2612                "type": "record",
2613                "name": "MapEntry",
2614                "fields": [
2615                    {"name": "key", "type": "string"},
2616                    {"name": "value", "type": "Value"}
2617                ]
2618            }
2619        });
2620        Schema::parse_str(&schema.to_string())
2621            .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2622    }) {
2623        Ok(schema) => Ok(schema),
2624        Err(message) => Err(Error::Codec(message.clone())),
2625    }
2626}
2627
2628#[derive(Clone, Debug)]
2629pub struct Client {
2630    http: reqwest::Client,
2631    base_url: String,
2632    token: Option<String>,
2633    control_token: Option<String>,
2634    worker_token: Option<String>,
2635    namespace: String,
2636    max_external_payload_bytes: usize,
2637    worker_storage_admission: Option<WorkerStorageAdmission>,
2638    runtime_upload_policy: Arc<Mutex<runtime_uploads::PolicyCache>>,
2639}
2640
2641impl Client {
2642    pub fn new(base_url: impl Into<String>) -> Result<Self> {
2643        Self::builder(base_url).build()
2644    }
2645
2646    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2647        ClientBuilder {
2648            base_url: base_url.into(),
2649            token: None,
2650            control_token: None,
2651            worker_token: None,
2652            namespace: "default".to_string(),
2653            timeout: Duration::from_secs(60),
2654            max_external_payload_bytes: 64 * 1024 * 1024,
2655        }
2656    }
2657
2658    pub async fn health(&self) -> Result<Value> {
2659        self.request_json(
2660            reqwest::Method::GET,
2661            "/health",
2662            RequestProtocol::ControlPlane,
2663            Option::<&Value>::None,
2664        )
2665        .await
2666    }
2667
2668    pub async fn cluster_info(&self) -> Result<Value> {
2669        self.request_json(
2670            reqwest::Method::GET,
2671            "/cluster/info",
2672            RequestProtocol::ControlPlane,
2673            Option::<&Value>::None,
2674        )
2675        .await
2676    }
2677
2678    pub async fn start_workflow<T: Serialize>(
2679        &self,
2680        workflow_type: &str,
2681        task_queue: &str,
2682        workflow_id: &str,
2683        input: T,
2684    ) -> Result<WorkflowHandle> {
2685        self.start_workflow_with_options(
2686            workflow_type,
2687            task_queue,
2688            workflow_id,
2689            WorkflowStartOptions::default(),
2690            input,
2691        )
2692        .await
2693    }
2694
2695    /// Start a workflow with explicit server-enforced execution and run
2696    /// deadlines.
2697    pub async fn start_workflow_with_options<T: Serialize>(
2698        &self,
2699        workflow_type: &str,
2700        task_queue: &str,
2701        workflow_id: &str,
2702        options: WorkflowStartOptions,
2703        input: T,
2704    ) -> Result<WorkflowHandle> {
2705        options.validate()?;
2706        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2707        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2708        let body = json!({
2709            "workflow_id": workflow_id,
2710            "workflow_type": workflow_type,
2711            "task_queue": task_queue,
2712            "input": input_envelope,
2713            "execution_timeout_seconds": options.execution_timeout_seconds,
2714            "run_timeout_seconds": options.run_timeout_seconds
2715        });
2716
2717        let data: Value = self
2718            .request_json(
2719                reqwest::Method::POST,
2720                "/workflows",
2721                RequestProtocol::ControlPlane,
2722                Some(&body),
2723            )
2724            .await?;
2725
2726        Ok(WorkflowHandle {
2727            client: self.clone(),
2728            workflow_id: data
2729                .get("workflow_id")
2730                .and_then(Value::as_str)
2731                .unwrap_or(workflow_id)
2732                .to_string(),
2733            run_id: data
2734                .get("run_id")
2735                .and_then(Value::as_str)
2736                .map(str::to_string),
2737            workflow_type: data
2738                .get("workflow_type")
2739                .and_then(Value::as_str)
2740                .unwrap_or(workflow_type)
2741                .to_string(),
2742        })
2743    }
2744
2745    pub async fn signal_workflow<T: Serialize>(
2746        &self,
2747        workflow_id: &str,
2748        signal_name: &str,
2749        input: T,
2750    ) -> Result<Value> {
2751        self.signal_workflow_target(workflow_id, None, signal_name, input)
2752            .await
2753    }
2754
2755    /// Append one idempotently identified item to an instance-scoped input stream.
2756    pub async fn append_message_stream<T: Serialize>(
2757        &self,
2758        workflow_id: &str,
2759        stream_name: &str,
2760        message_id: &str,
2761        input: T,
2762    ) -> Result<Value> {
2763        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2764        let body = json!({
2765            "message_id": message_id,
2766            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2767        });
2768        self.request_json(
2769            reqwest::Method::POST,
2770            &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2771            RequestProtocol::ControlPlane,
2772            Some(&body),
2773        )
2774        .await
2775    }
2776
2777    /// Signal only if `run_id` is still the current run for this instance.
2778    pub async fn signal_workflow_run<T: Serialize>(
2779        &self,
2780        workflow_id: &str,
2781        run_id: &str,
2782        signal_name: &str,
2783        input: T,
2784    ) -> Result<Value> {
2785        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2786            .await
2787    }
2788
2789    async fn signal_workflow_target<T: Serialize>(
2790        &self,
2791        workflow_id: &str,
2792        run_id: Option<&str>,
2793        signal_name: &str,
2794        input: T,
2795    ) -> Result<Value> {
2796        validate_user_signal_name(signal_name)?;
2797        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2798        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2799        let body = json!({
2800            "input": input_envelope
2801        });
2802        let path = match run_id {
2803            Some(run_id) => {
2804                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2805            }
2806            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2807        };
2808        self.request_json(
2809            reqwest::Method::POST,
2810            &path,
2811            RequestProtocol::ControlPlane,
2812            Some(&body),
2813        )
2814        .await
2815    }
2816
2817    /// Request cooperative cancellation of the current run for an instance.
2818    pub async fn cancel_workflow(
2819        &self,
2820        workflow_id: &str,
2821        options: WorkflowCommandOptions,
2822    ) -> Result<WorkflowCommandResult> {
2823        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2824            .await
2825    }
2826
2827    /// Request cooperative cancellation only if `run_id` is still current.
2828    pub async fn cancel_workflow_run(
2829        &self,
2830        workflow_id: &str,
2831        run_id: &str,
2832        options: WorkflowCommandOptions,
2833    ) -> Result<WorkflowCommandResult> {
2834        self.workflow_command(
2835            workflow_id,
2836            Some(run_id),
2837            WorkflowCommandKind::Cancel,
2838            options,
2839        )
2840        .await
2841    }
2842
2843    /// Forcefully terminate the current run for an instance.
2844    pub async fn terminate_workflow(
2845        &self,
2846        workflow_id: &str,
2847        options: WorkflowCommandOptions,
2848    ) -> Result<WorkflowCommandResult> {
2849        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2850            .await
2851    }
2852
2853    /// Forcefully terminate only if `run_id` is still current.
2854    pub async fn terminate_workflow_run(
2855        &self,
2856        workflow_id: &str,
2857        run_id: &str,
2858        options: WorkflowCommandOptions,
2859    ) -> Result<WorkflowCommandResult> {
2860        self.workflow_command(
2861            workflow_id,
2862            Some(run_id),
2863            WorkflowCommandKind::Terminate,
2864            options,
2865        )
2866        .await
2867    }
2868
2869    /// Continue a failed run from its recorded activity failure boundary.
2870    pub async fn redrive_workflow_run(
2871        &self,
2872        workflow_id: &str,
2873        failed_run_id: &str,
2874        request_id: Option<&str>,
2875    ) -> Result<WorkflowRedriveResult> {
2876        let body = request_id
2877            .map(|id| json!({"request_id": id}))
2878            .unwrap_or_else(|| json!({}));
2879        let data: Value = self
2880            .request_json(
2881                reqwest::Method::POST,
2882                &format!("/workflows/{workflow_id}/runs/{failed_run_id}/redrive"),
2883                RequestProtocol::ControlPlane,
2884                Some(&body),
2885            )
2886            .await?;
2887        if data.get("command_status").and_then(Value::as_str) != Some("accepted")
2888            || data.get("outcome").and_then(Value::as_str) != Some("redriven")
2889        {
2890            return Err(Error::Codec(
2891                "redrive response was not accepted".to_string(),
2892            ));
2893        }
2894        if data.get("workflow_id").and_then(Value::as_str) != Some(workflow_id)
2895            || data.get("continued_from_run_id").and_then(Value::as_str) != Some(failed_run_id)
2896        {
2897            return Err(Error::Codec(
2898                "redrive response does not match the requested source run".to_string(),
2899            ));
2900        }
2901        let run_id = data
2902            .get("run_id")
2903            .and_then(Value::as_str)
2904            .filter(|id| !id.is_empty() && *id != failed_run_id)
2905            .ok_or_else(|| {
2906                Error::Codec("redrive response is missing a distinct successor run_id".to_string())
2907            })?;
2908        let resume_step_sequence = data
2909            .get("resume_step_sequence")
2910            .and_then(Value::as_u64)
2911            .filter(|sequence| *sequence > 0)
2912            .ok_or_else(|| {
2913                Error::Codec("redrive response is missing resume_step_sequence".to_string())
2914            })?;
2915        Ok(WorkflowRedriveResult {
2916            workflow_id: workflow_id.to_string(),
2917            source_run_id: failed_run_id.to_string(),
2918            run_id: run_id.to_string(),
2919            resume_step_sequence,
2920            raw: data,
2921        })
2922    }
2923
2924    async fn workflow_command(
2925        &self,
2926        workflow_id: &str,
2927        run_id: Option<&str>,
2928        command: WorkflowCommandKind,
2929        options: WorkflowCommandOptions,
2930    ) -> Result<WorkflowCommandResult> {
2931        let path = match run_id {
2932            Some(run_id) => format!(
2933                "/workflows/{workflow_id}/runs/{run_id}/{}",
2934                command.as_str()
2935            ),
2936            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2937        };
2938        let data = match self
2939            .request_json(
2940                reqwest::Method::POST,
2941                &path,
2942                RequestProtocol::ControlPlane,
2943                Some(&options),
2944            )
2945            .await
2946        {
2947            Ok(data) => data,
2948            Err(Error::Http { status, body }) => {
2949                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2950                    command,
2951                    status,
2952                    body,
2953                    workflow_id,
2954                    run_id,
2955                )));
2956            }
2957            Err(error) => return Err(error),
2958        };
2959
2960        Ok(workflow_command_result(command, data, workflow_id, run_id))
2961    }
2962
2963    /// Execute a named, read-only query against a running or completed workflow.
2964    ///
2965    /// Arguments and results use the platform payload envelope. Server and
2966    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
2967    /// reason, HTTP status, and original response body.
2968    pub async fn query_workflow<T: Serialize>(
2969        &self,
2970        workflow_id: &str,
2971        query_name: &str,
2972        input: T,
2973    ) -> Result<Value> {
2974        self.query_workflow_target(workflow_id, None, query_name, input)
2975            .await
2976    }
2977
2978    /// Query only if `run_id` is still current, preventing accidental retargeting.
2979    pub async fn query_workflow_run<T: Serialize>(
2980        &self,
2981        workflow_id: &str,
2982        run_id: &str,
2983        query_name: &str,
2984        input: T,
2985    ) -> Result<Value> {
2986        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2987            .await
2988    }
2989
2990    /// Query a workflow and return the lossless fixed Avro Value result.
2991    pub async fn query_workflow_avro_value<T: Serialize>(
2992        &self,
2993        workflow_id: &str,
2994        query_name: &str,
2995        input: T,
2996    ) -> Result<AvroValue> {
2997        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2998            .await
2999    }
3000
3001    /// Query a selected run and return the lossless fixed Avro Value result.
3002    pub async fn query_workflow_run_avro_value<T: Serialize>(
3003        &self,
3004        workflow_id: &str,
3005        run_id: &str,
3006        query_name: &str,
3007        input: T,
3008    ) -> Result<AvroValue> {
3009        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
3010            .await
3011    }
3012
3013    async fn query_workflow_avro_value_target<T: Serialize>(
3014        &self,
3015        workflow_id: &str,
3016        run_id: Option<&str>,
3017        query_name: &str,
3018        input: T,
3019    ) -> Result<AvroValue> {
3020        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3021        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
3022        let path = match run_id {
3023            Some(run_id) => {
3024                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3025            }
3026            None => format!("/workflows/{workflow_id}/query/{query_name}"),
3027        };
3028        let response: Value = match self
3029            .request_json(
3030                reqwest::Method::POST,
3031                &path,
3032                RequestProtocol::ControlPlane,
3033                Some(&body),
3034            )
3035            .await
3036        {
3037            Ok(response) => response,
3038            Err(Error::Http { status, body }) => {
3039                return Err(Error::QueryFailed(query_failure(status, body)));
3040            }
3041            Err(error) => return Err(error),
3042        };
3043
3044        let envelope = response
3045            .get("result_envelope")
3046            .filter(|envelope| !envelope.is_null())
3047            .ok_or_else(|| {
3048                Error::Codec(
3049                    "missing_payload_envelope: typed query result requires result_envelope"
3050                        .to_string(),
3051                )
3052            })?;
3053        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3054    }
3055
3056    async fn query_workflow_target<T: Serialize>(
3057        &self,
3058        workflow_id: &str,
3059        run_id: Option<&str>,
3060        query_name: &str,
3061        input: T,
3062    ) -> Result<Value> {
3063        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3064        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
3065        let body = json!({
3066            "input": input_envelope
3067        });
3068        let path = match run_id {
3069            Some(run_id) => {
3070                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3071            }
3072            None => format!("/workflows/{workflow_id}/query/{query_name}"),
3073        };
3074        let response: Value = match self
3075            .request_json(
3076                reqwest::Method::POST,
3077                &path,
3078                RequestProtocol::ControlPlane,
3079                Some(&body),
3080            )
3081            .await
3082        {
3083            Ok(response) => response,
3084            Err(Error::Http { status, body }) => {
3085                return Err(Error::QueryFailed(query_failure(status, body)));
3086            }
3087            Err(error) => return Err(error),
3088        };
3089
3090        if let Some(envelope) = response
3091            .get("result_envelope")
3092            .filter(|envelope| !envelope.is_null())
3093        {
3094            return decode_wire_value(envelope, DEFAULT_CODEC);
3095        }
3096
3097        Ok(response.get("result").cloned().unwrap_or(Value::Null))
3098    }
3099
3100    /// Send a synchronous update using fixed Avro Value arguments.
3101    pub async fn update_workflow<T: Serialize>(
3102        &self,
3103        workflow_id: &str,
3104        update_name: &str,
3105        input: T,
3106        request_id: Option<&str>,
3107    ) -> Result<Value> {
3108        let response = self
3109            .update_workflow_response(workflow_id, update_name, input, request_id)
3110            .await?;
3111        if let Some(envelope) = response
3112            .get("result_envelope")
3113            .filter(|envelope| !envelope.is_null())
3114        {
3115            return decode_wire_value(envelope, DEFAULT_CODEC);
3116        }
3117        Ok(response.get("result").cloned().unwrap_or(response))
3118    }
3119
3120    /// Send a synchronous update and retain a bytes-capable Avro result.
3121    pub async fn update_workflow_avro_value<T: Serialize>(
3122        &self,
3123        workflow_id: &str,
3124        update_name: &str,
3125        input: T,
3126        request_id: Option<&str>,
3127    ) -> Result<AvroValue> {
3128        let response = self
3129            .update_workflow_response(workflow_id, update_name, input, request_id)
3130            .await?;
3131        let envelope = response
3132            .get("result_envelope")
3133            .filter(|envelope| !envelope.is_null())
3134            .ok_or_else(|| {
3135                Error::Codec(
3136                    "missing_payload_envelope: typed update result requires result_envelope"
3137                        .to_string(),
3138                )
3139            })?;
3140        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3141    }
3142
3143    async fn update_workflow_response<T: Serialize>(
3144        &self,
3145        workflow_id: &str,
3146        update_name: &str,
3147        input: T,
3148        request_id: Option<&str>,
3149    ) -> Result<Value> {
3150        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3151        let mut body = json!({
3152            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3153            "wait_for": "completed",
3154        });
3155        if let Some(request_id) = request_id {
3156            body["request_id"] = json!(request_id);
3157        }
3158        self.request_json(
3159            reqwest::Method::POST,
3160            &format!("/workflows/{workflow_id}/update/{update_name}"),
3161            RequestProtocol::ControlPlane,
3162            Some(&body),
3163        )
3164        .await
3165    }
3166
3167    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3168        let path = format!("/workflows/{workflow_id}");
3169        let mut data: WorkflowDescription = self
3170            .request_json(
3171                reqwest::Method::GET,
3172                &path,
3173                RequestProtocol::ControlPlane,
3174                Option::<&Value>::None,
3175            )
3176            .await?;
3177        data.decode_payloads()?;
3178        Ok(data)
3179    }
3180
3181    /// Describe one selected run, including historical terminal runs.
3182    pub async fn describe_workflow_run(
3183        &self,
3184        workflow_id: &str,
3185        run_id: &str,
3186    ) -> Result<WorkflowDescription> {
3187        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3188        let mut data: WorkflowDescription = self
3189            .request_json(
3190                reqwest::Method::GET,
3191                &path,
3192                RequestProtocol::ControlPlane,
3193                Option::<&Value>::None,
3194            )
3195            .await?;
3196        data.decode_payloads()?;
3197        Ok(data)
3198    }
3199
3200    fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3201        let mut path = format!(
3202            "/workflows/{}/runs/{}/streams",
3203            percent_encode_path_segment(workflow_id),
3204            percent_encode_path_segment(run_id),
3205        );
3206        if let Some(stream_name) = stream_name {
3207            path.push('/');
3208            path.push_str(&percent_encode_path_segment(stream_name));
3209        }
3210        path
3211    }
3212
3213    /// List the run-scoped output streams already opened by a workflow.
3214    pub async fn list_workflow_streams(
3215        &self,
3216        workflow_id: &str,
3217        run_id: &str,
3218    ) -> Result<Vec<WorkflowStreamDescription>> {
3219        let response: WorkflowStreamListResponse = self
3220            .request_json(
3221                reqwest::Method::GET,
3222                &Self::workflow_stream_path(workflow_id, run_id, None),
3223                RequestProtocol::ControlPlane,
3224                Option::<&Value>::None,
3225            )
3226            .await?;
3227        Ok(response.streams)
3228    }
3229
3230    /// Describe stream lifecycle, offsets, pending count, and terminal error.
3231    pub async fn describe_workflow_stream(
3232        &self,
3233        workflow_id: &str,
3234        run_id: &str,
3235        stream_name: &str,
3236    ) -> Result<WorkflowStreamDescription> {
3237        let response: WorkflowStreamDescriptionResponse = self
3238            .request_json(
3239                reqwest::Method::GET,
3240                &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3241                RequestProtocol::ControlPlane,
3242                Option::<&Value>::None,
3243            )
3244            .await?;
3245        Ok(response.stream)
3246    }
3247
3248    /// Read one bounded page beginning at a zero-based offset.
3249    ///
3250    /// Delivery is at least once: persist `next_offset` only after processing
3251    /// the page. The future is cancellation-safe; dropping it cancels the
3252    /// in-flight request. Long polling is capped at 60 seconds by the SDK and
3253    /// service contract.
3254    pub async fn subscribe_workflow_stream(
3255        &self,
3256        workflow_id: &str,
3257        run_id: &str,
3258        stream_name: &str,
3259        from_offset: u64,
3260        max_items: usize,
3261        wait: Duration,
3262    ) -> Result<WorkflowStreamPage> {
3263        let max_items = max_items.clamp(1, 500);
3264        let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3265        let path = format!(
3266            "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3267            Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3268        );
3269        let response: WorkflowStreamPageResponse = self
3270            .request_json_with_timeout(
3271                reqwest::Method::GET,
3272                &path,
3273                RequestProtocol::ControlPlane,
3274                Option::<&Value>::None,
3275                Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3276            )
3277            .await?;
3278
3279        let items = response
3280            .items
3281            .into_iter()
3282            .map(|raw| {
3283                let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3284                let envelope = raw.get("payload").cloned();
3285                let payload = envelope
3286                    .as_ref()
3287                    .filter(|value| value.get("blob").is_some())
3288                    .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3289                    .transpose()?
3290                    .map(AvroValue::into_json)
3291                    .transpose()?;
3292                Ok(WorkflowStreamItem {
3293                    offset,
3294                    payload,
3295                    payload_envelope: envelope,
3296                    payload_reference: raw
3297                        .get("payload_reference")
3298                        .and_then(Value::as_str)
3299                        .map(str::to_string),
3300                    payload_codec: raw
3301                        .get("payload_codec")
3302                        .and_then(Value::as_str)
3303                        .map(str::to_string),
3304                    idempotency_key: raw
3305                        .get("idempotency_key")
3306                        .and_then(Value::as_str)
3307                        .map(str::to_string),
3308                    item_type: raw
3309                        .get("item_type")
3310                        .and_then(Value::as_str)
3311                        .map(str::to_string),
3312                    content_type: raw
3313                        .get("content_type")
3314                        .and_then(Value::as_str)
3315                        .map(str::to_string),
3316                    origin: raw
3317                        .get("origin")
3318                        .and_then(Value::as_str)
3319                        .map(str::to_string),
3320                    origin_reference: raw
3321                        .get("origin_reference")
3322                        .and_then(Value::as_str)
3323                        .map(str::to_string),
3324                    emitted_at: raw
3325                        .get("emitted_at")
3326                        .and_then(Value::as_str)
3327                        .map(str::to_string),
3328                    raw,
3329                })
3330            })
3331            .collect::<Result<Vec<_>>>()?;
3332        Ok(WorkflowStreamPage {
3333            stream: response.stream,
3334            items,
3335            next_offset: response.next_offset,
3336            terminal: response.terminal,
3337        })
3338    }
3339
3340    /// Append inline Avro envelopes or opaque external payload references.
3341    pub async fn append_workflow_stream(
3342        &self,
3343        workflow_id: &str,
3344        run_id: &str,
3345        stream_name: &str,
3346        items: &[WorkflowStreamAppendItem],
3347        max_pending_items: Option<u64>,
3348    ) -> Result<WorkflowStreamAppendResult> {
3349        if items.is_empty() {
3350            return Err(Error::Codec(
3351                "workflow_stream_items_empty: append requires at least one item".to_string(),
3352            ));
3353        }
3354        let mut body = json!({
3355            "items": items
3356                .iter()
3357                .map(|item| item.wire_value(None))
3358                .collect::<Vec<_>>(),
3359        });
3360        if let Some(max_pending_items) = max_pending_items {
3361            if max_pending_items == 0 {
3362                return Err(Error::Codec(
3363                    "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3364                        .to_string(),
3365                ));
3366            }
3367            body["max_pending_items"] = json!(max_pending_items);
3368        }
3369        let response: WorkflowStreamAppendResponse = self
3370            .request_json(
3371                reqwest::Method::POST,
3372                &format!(
3373                    "{}/items",
3374                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3375                ),
3376                RequestProtocol::ControlPlane,
3377                Some(&body),
3378            )
3379            .await?;
3380        Ok(WorkflowStreamAppendResult {
3381            stream: response.stream,
3382            accepted_offsets: response.accepted_offsets,
3383            accepted: response.accepted,
3384            deduped: response.deduped,
3385        })
3386    }
3387
3388    /// Close a stream, or mark it errored when `error_reason` is supplied.
3389    pub async fn close_workflow_stream(
3390        &self,
3391        workflow_id: &str,
3392        run_id: &str,
3393        stream_name: &str,
3394        error_reason: Option<&str>,
3395        retention_seconds: Option<u64>,
3396    ) -> Result<WorkflowStreamDescription> {
3397        let mut body = json!({});
3398        if let Some(error_reason) = error_reason {
3399            body["error_reason"] = json!(error_reason);
3400        }
3401        if let Some(retention_seconds) = retention_seconds {
3402            if retention_seconds == 0 {
3403                return Err(Error::Codec(
3404                    "workflow_stream_retention_invalid: retention_seconds must be positive"
3405                        .to_string(),
3406                ));
3407            }
3408            body["retention_seconds"] = json!(retention_seconds);
3409        }
3410        let response: WorkflowStreamDescriptionResponse = self
3411            .request_json(
3412                reqwest::Method::POST,
3413                &format!(
3414                    "{}/close",
3415                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3416                ),
3417                RequestProtocol::ControlPlane,
3418                Some(&body),
3419            )
3420            .await?;
3421        Ok(response.stream)
3422    }
3423
3424    pub async fn register_worker(
3425        &self,
3426        worker_id: &str,
3427        task_queue: &str,
3428        supported_workflow_types: Vec<String>,
3429        supported_activity_types: Vec<String>,
3430        max_concurrent_workflow_tasks: usize,
3431        max_concurrent_activity_tasks: usize,
3432    ) -> Result<RegisterWorkerResponse> {
3433        self.register_worker_with_capabilities(
3434            worker_id,
3435            task_queue,
3436            supported_workflow_types,
3437            supported_activity_types,
3438            max_concurrent_workflow_tasks,
3439            max_concurrent_activity_tasks,
3440            Vec::new(),
3441        )
3442        .await
3443    }
3444
3445    /// Register a worker and explicitly advertise additive worker capabilities.
3446    pub async fn register_worker_with_capabilities(
3447        &self,
3448        worker_id: &str,
3449        task_queue: &str,
3450        supported_workflow_types: Vec<String>,
3451        supported_activity_types: Vec<String>,
3452        max_concurrent_workflow_tasks: usize,
3453        max_concurrent_activity_tasks: usize,
3454        capabilities: Vec<String>,
3455    ) -> Result<RegisterWorkerResponse> {
3456        self.register_worker_with_command_contracts(
3457            worker_id,
3458            task_queue,
3459            supported_workflow_types,
3460            supported_activity_types,
3461            max_concurrent_workflow_tasks,
3462            max_concurrent_activity_tasks,
3463            capabilities,
3464            Value::Object(serde_json::Map::new()),
3465        )
3466        .await
3467    }
3468
3469    /// Register a worker and advertise its named query and update handlers.
3470    ///
3471    /// This Rust SDK cannot execute synchronous pre-accept update validation,
3472    /// so a workflow contract with a non-empty or malformed
3473    /// `update_validators` declaration returns
3474    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
3475    #[allow(clippy::too_many_arguments)]
3476    pub async fn register_worker_with_command_contracts(
3477        &self,
3478        worker_id: &str,
3479        task_queue: &str,
3480        supported_workflow_types: Vec<String>,
3481        supported_activity_types: Vec<String>,
3482        max_concurrent_workflow_tasks: usize,
3483        max_concurrent_activity_tasks: usize,
3484        capabilities: Vec<String>,
3485        workflow_command_contracts: Value,
3486    ) -> Result<RegisterWorkerResponse> {
3487        self.register_worker_with_definition_fingerprints(
3488            worker_id,
3489            task_queue,
3490            supported_workflow_types,
3491            supported_activity_types,
3492            max_concurrent_workflow_tasks,
3493            max_concurrent_activity_tasks,
3494            capabilities,
3495            workflow_command_contracts,
3496            None,
3497        )
3498        .await
3499    }
3500
3501    #[allow(clippy::too_many_arguments)]
3502    async fn register_worker_with_definition_fingerprints(
3503        &self,
3504        worker_id: &str,
3505        task_queue: &str,
3506        supported_workflow_types: Vec<String>,
3507        supported_activity_types: Vec<String>,
3508        max_concurrent_workflow_tasks: usize,
3509        max_concurrent_activity_tasks: usize,
3510        capabilities: Vec<String>,
3511        workflow_command_contracts: Value,
3512        workflow_definition_fingerprints: Option<BTreeMap<String, String>>,
3513    ) -> Result<RegisterWorkerResponse> {
3514        if let Some(contracts) = workflow_command_contracts.as_object() {
3515            for (workflow_type, contract) in contracts {
3516                let Some(update_validators) = contract.get("update_validators") else {
3517                    continue;
3518                };
3519                if !update_validators
3520                    .as_array()
3521                    .is_some_and(|validators| validators.is_empty())
3522                {
3523                    return Err(Error::UnsupportedUpdateValidators {
3524                        workflow_type: workflow_type.clone(),
3525                    });
3526                }
3527            }
3528        }
3529
3530        let mut body = json!({
3531            "worker_id": worker_id,
3532            "task_queue": task_queue,
3533            "runtime": "rust",
3534            "sdk_version": SDK_VERSION,
3535            "supported_workflow_types": supported_workflow_types,
3536            "supported_activity_types": supported_activity_types,
3537            "capabilities": capabilities,
3538            "capability_manifest": portable_worker_affinity_capability_manifest(),
3539            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3540            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3541        });
3542        if workflow_command_contracts
3543            .as_object()
3544            .is_some_and(|contracts| !contracts.is_empty())
3545        {
3546            body["workflow_command_contracts"] = workflow_command_contracts;
3547        }
3548        if let Some(fingerprints) = workflow_definition_fingerprints {
3549            body["workflow_definition_fingerprints"] = json!(fingerprints);
3550        }
3551
3552        self.request_json(
3553            reqwest::Method::POST,
3554            "/worker/register",
3555            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3556            Some(&body),
3557        )
3558        .await
3559    }
3560
3561    /// Gracefully remove one worker's registration through the worker plane.
3562    ///
3563    /// This operation is separate from operator-facing worker management. It
3564    /// uses worker-protocol authentication and returns the server's lease
3565    /// recovery result.
3566    pub async fn deregister_worker_registration(
3567        &self,
3568        worker_id: &str,
3569    ) -> Result<WorkerDeregistrationEnvelope> {
3570        let path = format!(
3571            "/worker/registrations/{}",
3572            percent_encode_path_segment(worker_id)
3573        );
3574        self.request_json(
3575            reqwest::Method::DELETE,
3576            &path,
3577            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3578            Option::<&Value>::None,
3579        )
3580        .await
3581    }
3582
3583    /// Long-poll for an ephemeral, read-only workflow query task.
3584    pub async fn poll_query_task(
3585        &self,
3586        worker_id: &str,
3587        task_queue: &str,
3588        timeout: Duration,
3589    ) -> Result<Option<QueryTask>> {
3590        Ok(self
3591            .poll_query_task_response(worker_id, task_queue, timeout)
3592            .await?
3593            .task)
3594    }
3595
3596    /// Poll a query task while preserving server stop and drain metadata.
3597    pub async fn poll_query_task_response(
3598        &self,
3599        worker_id: &str,
3600        task_queue: &str,
3601        timeout: Duration,
3602    ) -> Result<PollQueryTaskResponse> {
3603        let poll_request_id = unique_request_id("rust-query-poll");
3604        self.poll_query_task_response_with_request_id(
3605            worker_id,
3606            task_queue,
3607            timeout,
3608            &poll_request_id,
3609            1,
3610        )
3611        .await
3612    }
3613
3614    async fn poll_query_task_response_with_request_id(
3615        &self,
3616        worker_id: &str,
3617        task_queue: &str,
3618        timeout: Duration,
3619        poll_request_id: &str,
3620        transport_retries: usize,
3621    ) -> Result<PollQueryTaskResponse> {
3622        let timeout_seconds = long_poll_timeout_seconds(timeout);
3623        let body = json!({
3624            "worker_id": worker_id,
3625            "task_queue": task_queue,
3626            "poll_request_id": poll_request_id,
3627            "timeout_seconds": timeout_seconds,
3628        });
3629        self.poll_request_json(
3630            "/worker/query-tasks/poll",
3631            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3632            &body,
3633            timeout + Duration::from_secs(5),
3634            transport_retries,
3635        )
3636        .await
3637    }
3638
3639    /// Complete a query task without appending workflow history.
3640    pub async fn complete_query_task<T: Serialize>(
3641        &self,
3642        query_task_id: &str,
3643        lease_owner: &str,
3644        query_task_attempt: u64,
3645        result: T,
3646        codec: &str,
3647    ) -> Result<Value> {
3648        let typed_result = AvroValue::from_serialize(&result)?;
3649        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3650        self.complete_query_task_with_envelope(
3651            query_task_id,
3652            lease_owner,
3653            query_task_attempt,
3654            typed_result.into_json()?,
3655            result_envelope,
3656        )
3657        .await
3658    }
3659
3660    async fn complete_query_task_with_envelope(
3661        &self,
3662        query_task_id: &str,
3663        lease_owner: &str,
3664        query_task_attempt: u64,
3665        result: Value,
3666        result_envelope: Value,
3667    ) -> Result<Value> {
3668        let body = json!({
3669            "lease_owner": lease_owner,
3670            "query_task_attempt": query_task_attempt,
3671            "result": result,
3672            "result_envelope": result_envelope,
3673        });
3674        let path = format!("/worker/query-tasks/{query_task_id}/complete");
3675        let response = self
3676            .request_json(
3677                reqwest::Method::POST,
3678                &path,
3679                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3680                Some(&body),
3681            )
3682            .await;
3683        query_task_response(response)
3684    }
3685
3686    /// Report a stable machine-readable query-task failure.
3687    pub async fn fail_query_task(
3688        &self,
3689        query_task_id: &str,
3690        lease_owner: &str,
3691        query_task_attempt: u64,
3692        message: impl Into<String>,
3693        reason: impl Into<String>,
3694        failure_type: impl Into<String>,
3695    ) -> Result<Value> {
3696        let body = json!({
3697            "lease_owner": lease_owner,
3698            "query_task_attempt": query_task_attempt,
3699            "failure": {
3700                "message": message.into(),
3701                "reason": reason.into(),
3702                "type": failure_type.into(),
3703            }
3704        });
3705        let path = format!("/worker/query-tasks/{query_task_id}/fail");
3706        let response = self
3707            .request_json(
3708                reqwest::Method::POST,
3709                &path,
3710                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3711                Some(&body),
3712            )
3713            .await;
3714        query_task_response(response)
3715    }
3716
3717    pub async fn heartbeat_worker(
3718        &self,
3719        worker_id: &str,
3720        workflow_available: usize,
3721        activity_available: usize,
3722    ) -> Result<Value> {
3723        let body = json!({
3724            "worker_id": worker_id,
3725            "task_slots": {
3726                "workflow_available": workflow_available,
3727                "activity_available": activity_available
3728            },
3729            "process_metrics": {
3730                "process_id": std::process::id(),
3731                "process_uptime_seconds": 0
3732            }
3733        });
3734
3735        self.request_json(
3736            reqwest::Method::POST,
3737            "/worker/heartbeat",
3738            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3739            Some(&body),
3740        )
3741        .await
3742    }
3743
3744    pub async fn poll_workflow_task(
3745        &self,
3746        worker_id: &str,
3747        task_queue: &str,
3748        timeout: Duration,
3749    ) -> Result<Option<WorkflowTask>> {
3750        Ok(self
3751            .poll_workflow_task_response(worker_id, task_queue, timeout)
3752            .await?
3753            .task)
3754    }
3755
3756    pub async fn poll_workflow_task_response(
3757        &self,
3758        worker_id: &str,
3759        task_queue: &str,
3760        timeout: Duration,
3761    ) -> Result<PollWorkflowTaskResponse> {
3762        let poll_request_id = unique_request_id("rust-workflow-poll");
3763        self.poll_workflow_task_response_with_request_id(
3764            worker_id,
3765            task_queue,
3766            timeout,
3767            &poll_request_id,
3768            1,
3769        )
3770        .await
3771    }
3772
3773    async fn poll_workflow_task_response_with_request_id(
3774        &self,
3775        worker_id: &str,
3776        task_queue: &str,
3777        timeout: Duration,
3778        poll_request_id: &str,
3779        transport_retries: usize,
3780    ) -> Result<PollWorkflowTaskResponse> {
3781        let body = json!({
3782            "worker_id": worker_id,
3783            "task_queue": task_queue,
3784            "poll_request_id": poll_request_id,
3785            "timeout_seconds": long_poll_timeout_seconds(timeout),
3786        });
3787        let mut data: PollWorkflowTaskResponse = self
3788            .poll_request_json(
3789                "/worker/workflow-tasks/poll",
3790                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3791                &body,
3792                timeout + Duration::from_secs(5),
3793                transport_retries,
3794            )
3795            .await?;
3796
3797        if let Some(task) = data.task.as_mut() {
3798            self.fetch_remaining_workflow_history(worker_id, task)
3799                .await?;
3800        }
3801
3802        Ok(data)
3803    }
3804
3805    async fn fetch_remaining_workflow_history(
3806        &self,
3807        worker_id: &str,
3808        task: &mut WorkflowTask,
3809    ) -> Result<()> {
3810        let mut next_token = task.next_history_page_token.clone();
3811
3812        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3813            let lease_owner = task
3814                .lease_owner
3815                .clone()
3816                .unwrap_or_else(|| worker_id.to_string());
3817            let page = self
3818                .workflow_task_history_page(
3819                    &task.task_id,
3820                    &lease_owner,
3821                    task.workflow_task_attempt,
3822                    &token,
3823                )
3824                .await?;
3825
3826            task.append_history_page(page);
3827
3828            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3829                return Err(Error::Codec(
3830                    "workflow history pagination returned the same page token".to_string(),
3831                ));
3832            }
3833
3834            next_token = task.next_history_page_token.clone();
3835        }
3836
3837        Ok(())
3838    }
3839
3840    async fn workflow_task_history_page(
3841        &self,
3842        task_id: &str,
3843        lease_owner: &str,
3844        workflow_task_attempt: u64,
3845        next_history_page_token: &str,
3846    ) -> Result<WorkflowTaskHistoryPage> {
3847        let body = json!({
3848            "lease_owner": lease_owner,
3849            "workflow_task_attempt": workflow_task_attempt,
3850            "next_history_page_token": next_history_page_token
3851        });
3852        let path = format!("/worker/workflow-tasks/{task_id}/history");
3853
3854        self.request_json(
3855            reqwest::Method::POST,
3856            &path,
3857            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3858            Some(&body),
3859        )
3860        .await
3861    }
3862
3863    pub async fn complete_workflow_task(
3864        &self,
3865        task_id: &str,
3866        lease_owner: &str,
3867        workflow_task_attempt: u64,
3868        commands: Vec<Value>,
3869    ) -> Result<Value> {
3870        self.complete_workflow_task_with_message_streams(
3871            task_id,
3872            lease_owner,
3873            workflow_task_attempt,
3874            commands,
3875            Vec::new(),
3876            Vec::new(),
3877        )
3878        .await
3879    }
3880
3881    async fn complete_workflow_task_with_message_streams(
3882        &self,
3883        task_id: &str,
3884        lease_owner: &str,
3885        workflow_task_attempt: u64,
3886        commands: Vec<Value>,
3887        message_stream_cursors: Vec<Value>,
3888        message_stream_waits: Vec<Value>,
3889    ) -> Result<Value> {
3890        validate_workflow_task_commands(&commands)?;
3891        let has_message_stream_metadata =
3892            !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3893        if has_message_stream_metadata
3894            && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3895        {
3896            return Err(Error::Codec(
3897                "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3898                    .to_string(),
3899            ));
3900        }
3901        let protocol_version = workflow_completion_protocol_version_with_message_streams(
3902            &commands,
3903            has_message_stream_metadata,
3904        );
3905        let mut body = json!({
3906            "lease_owner": lease_owner,
3907            "workflow_task_attempt": workflow_task_attempt,
3908            "commands": commands
3909        });
3910        if !message_stream_cursors.is_empty() {
3911            body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3912        }
3913        if !message_stream_waits.is_empty() {
3914            body["message_stream_waits"] = Value::Array(message_stream_waits);
3915        }
3916        let path = format!("/worker/workflow-tasks/{task_id}/complete");
3917        self.request_json(
3918            reqwest::Method::POST,
3919            &path,
3920            RequestProtocol::Worker(protocol_version),
3921            Some(&body),
3922        )
3923        .await
3924    }
3925
3926    pub async fn fail_workflow_task(
3927        &self,
3928        task_id: &str,
3929        lease_owner: &str,
3930        workflow_task_attempt: u64,
3931        message: impl Into<String>,
3932    ) -> Result<Value> {
3933        self.fail_workflow_task_with_type(
3934            task_id,
3935            lease_owner,
3936            workflow_task_attempt,
3937            message,
3938            "RustWorkflowTaskFailure",
3939        )
3940        .await
3941    }
3942
3943    async fn fail_workflow_task_with_type(
3944        &self,
3945        task_id: &str,
3946        lease_owner: &str,
3947        workflow_task_attempt: u64,
3948        message: impl Into<String>,
3949        failure_type: &str,
3950    ) -> Result<Value> {
3951        let body = json!({
3952            "lease_owner": lease_owner,
3953            "workflow_task_attempt": workflow_task_attempt,
3954            "failure": {
3955                "message": message.into(),
3956                "type": failure_type
3957            }
3958        });
3959        let path = format!("/worker/workflow-tasks/{task_id}/fail");
3960        self.request_json(
3961            reqwest::Method::POST,
3962            &path,
3963            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3964            Some(&body),
3965        )
3966        .await
3967    }
3968
3969    pub async fn poll_activity_task(
3970        &self,
3971        worker_id: &str,
3972        task_queue: &str,
3973        timeout: Duration,
3974    ) -> Result<Option<ActivityTask>> {
3975        Ok(self
3976            .poll_activity_task_response(worker_id, task_queue, timeout)
3977            .await?
3978            .task)
3979    }
3980
3981    /// Poll an activity task while preserving server stop and drain metadata.
3982    pub async fn poll_activity_task_response(
3983        &self,
3984        worker_id: &str,
3985        task_queue: &str,
3986        timeout: Duration,
3987    ) -> Result<PollActivityTaskResponse> {
3988        let poll_request_id = unique_request_id("rust-activity-poll");
3989        self.poll_activity_task_response_with_request_id(
3990            worker_id,
3991            task_queue,
3992            timeout,
3993            &poll_request_id,
3994            1,
3995        )
3996        .await
3997    }
3998
3999    async fn poll_activity_task_response_with_request_id(
4000        &self,
4001        worker_id: &str,
4002        task_queue: &str,
4003        timeout: Duration,
4004        poll_request_id: &str,
4005        transport_retries: usize,
4006    ) -> Result<PollActivityTaskResponse> {
4007        let body = json!({
4008            "worker_id": worker_id,
4009            "task_queue": task_queue,
4010            "poll_request_id": poll_request_id,
4011            "timeout_seconds": long_poll_timeout_seconds(timeout),
4012        });
4013        let data: PollActivityTaskResponse = self
4014            .poll_request_json(
4015                "/worker/activity-tasks/poll",
4016                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4017                &body,
4018                timeout + Duration::from_secs(5),
4019                transport_retries,
4020            )
4021            .await?;
4022        Ok(data)
4023    }
4024
4025    pub async fn complete_activity_task<T: Serialize>(
4026        &self,
4027        task_id: &str,
4028        activity_attempt_id: &str,
4029        lease_owner: &str,
4030        result: T,
4031        codec: &str,
4032    ) -> Result<Value> {
4033        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
4034        let body = json!({
4035            "activity_attempt_id": activity_attempt_id,
4036            "lease_owner": lease_owner,
4037            "result": result
4038        });
4039        let path = format!("/worker/activity-tasks/{task_id}/complete");
4040        activity_task_response(
4041            self.request_json(
4042                reqwest::Method::POST,
4043                &path,
4044                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4045                Some(&body),
4046            )
4047            .await,
4048            "complete",
4049            task_id,
4050            activity_attempt_id,
4051        )
4052    }
4053
4054    pub async fn fail_activity_task(
4055        &self,
4056        task_id: &str,
4057        activity_attempt_id: &str,
4058        lease_owner: &str,
4059        message: impl Into<String>,
4060        non_retryable: bool,
4061    ) -> Result<Value> {
4062        let body = json!({
4063            "activity_attempt_id": activity_attempt_id,
4064            "lease_owner": lease_owner,
4065            "failure": {
4066                "message": message.into(),
4067                "type": "RustActivityFailure",
4068                "non_retryable": non_retryable
4069            }
4070        });
4071        let path = format!("/worker/activity-tasks/{task_id}/fail");
4072        activity_task_response(
4073            self.request_json(
4074                reqwest::Method::POST,
4075                &path,
4076                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4077                Some(&body),
4078            )
4079            .await,
4080            "fail",
4081            task_id,
4082            activity_attempt_id,
4083        )
4084    }
4085
4086    pub async fn heartbeat_activity_task<T: Serialize>(
4087        &self,
4088        task_id: &str,
4089        activity_attempt_id: &str,
4090        lease_owner: &str,
4091        details: T,
4092    ) -> Result<ActivityHeartbeatResponse> {
4093        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
4094        let body = json!({
4095            "activity_attempt_id": activity_attempt_id,
4096            "lease_owner": lease_owner,
4097            "details": details
4098        });
4099        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4100        activity_task_response(
4101            self.request_json(
4102                reqwest::Method::POST,
4103                &path,
4104                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4105                Some(&body),
4106            )
4107            .await,
4108            "heartbeat",
4109            task_id,
4110            activity_attempt_id,
4111        )
4112    }
4113
4114    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4115        &self,
4116        method: reqwest::Method,
4117        path: &str,
4118        protocol: RequestProtocol,
4119        body: Option<&B>,
4120    ) -> Result<T> {
4121        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4122            .await
4123    }
4124
4125    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4126        &self,
4127        method: reqwest::Method,
4128        path: &str,
4129        protocol: RequestProtocol,
4130        body: Option<&B>,
4131        timeout: Duration,
4132    ) -> Result<T> {
4133        let auth_token = self.auth_token(protocol)?;
4134        let mut request = self
4135            .http
4136            .request(method.clone(), format!("{}/api{}", self.base_url, path))
4137            .timeout(timeout)
4138            .header(reqwest::header::ACCEPT, "application/json")
4139            .header(reqwest::header::CONTENT_TYPE, "application/json")
4140            .header("X-Namespace", &self.namespace);
4141
4142        match protocol {
4143            RequestProtocol::Worker(version) => {
4144                request = request.header("X-Durable-Workflow-Protocol-Version", version);
4145            }
4146            RequestProtocol::ControlPlane => {
4147                request = request.header(
4148                    "X-Durable-Workflow-Control-Plane-Version",
4149                    CONTROL_PLANE_VERSION,
4150                );
4151            }
4152        }
4153
4154        if let Some(token) = auth_token {
4155            request = request.bearer_auth(token);
4156        }
4157
4158        if let Some(body) = body {
4159            let mut body = serde_json::to_value(body)?;
4160            if matches!(
4161                method,
4162                reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH
4163            ) {
4164                self.externalize_runtime_payloads(&mut body, path, protocol)
4165                    .await?;
4166            }
4167            request = request.json(&body);
4168        }
4169
4170        let request = request.build()?;
4171        let poll_request_id = path.ends_with("/poll").then(|| {
4172            request
4173                .body()
4174                .and_then(reqwest::Body::as_bytes)
4175                .and_then(|body| serde_json::from_slice::<Value>(body).ok())
4176                .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4177                .unwrap_or_default()
4178        });
4179        let mut storage_retries = 0_usize;
4180
4181        loop {
4182            // Keep the serialized body and lease/poll identity across admission pauses.
4183            let response = self
4184                .http
4185                .execute(request.try_clone().ok_or_else(|| {
4186                    Error::WorkerLoop("worker request body cannot be retried".to_string())
4187                })?)
4188                .await?;
4189            let status = response.status();
4190            let bytes = response.bytes().await?;
4191
4192            if !status.is_success() {
4193                let body = String::from_utf8_lossy(&bytes).to_string();
4194                if let Some(protocol) = protocol_failure(status, &body) {
4195                    return Err(Error::Protocol(protocol));
4196                }
4197                let error = Error::Http { status, body };
4198                if self
4199                    .wait_for_storage_admission(
4200                        &error,
4201                        protocol,
4202                        Some(path),
4203                        poll_request_id.as_deref(),
4204                        &mut storage_retries,
4205                    )
4206                    .await
4207                {
4208                    continue;
4209                }
4210                return Err(error);
4211            }
4212
4213            if bytes.is_empty() {
4214                return Ok(serde_json::from_value(Value::Null)?);
4215            }
4216
4217            let mut value: Value = serde_json::from_slice(&bytes)?;
4218            self.resolve_runtime_payloads(&mut value, path, protocol)
4219                .await?;
4220            return Ok(serde_json::from_value(value)?);
4221        }
4222    }
4223
4224    async fn wait_for_storage_admission(
4225        &self,
4226        error: &Error,
4227        protocol: RequestProtocol,
4228        path: Option<&str>,
4229        poll_request_id: Option<&str>,
4230        retries: &mut usize,
4231    ) -> bool {
4232        let Some(admission) = self
4233            .worker_storage_admission
4234            .as_ref()
4235            .filter(|_| matches!(protocol, RequestProtocol::Worker(_)))
4236        else {
4237            return false;
4238        };
4239        let Some(advertised_delay) = worker_storage_admission_retry_after(error, poll_request_id)
4240            .or_else(|| {
4241                path.and_then(|path| {
4242                    worker_backend_unavailable_retry_after(error, path, poll_request_id)
4243                })
4244            })
4245        else {
4246            return false;
4247        };
4248        *retries = retries.saturating_add(1);
4249        let delay = worker_retry_delay(admission.policy, *retries)
4250            .max(advertised_delay)
4251            .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4252        let deadline = tokio::time::Instant::now() + delay;
4253        loop {
4254            if admission.stop.load(Ordering::SeqCst) {
4255                return false;
4256            }
4257            let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
4258            if remaining.is_zero() {
4259                return true;
4260            }
4261            tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4262        }
4263    }
4264
4265    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4266        &self,
4267        path: &str,
4268        protocol: RequestProtocol,
4269        body: &B,
4270        timeout: Duration,
4271        max_retries: usize,
4272    ) -> Result<T> {
4273        let mut retries = 0;
4274
4275        loop {
4276            let response = self
4277                .request_json_with_timeout(
4278                    reqwest::Method::POST,
4279                    path,
4280                    protocol,
4281                    Some(body),
4282                    timeout,
4283                )
4284                .await;
4285
4286            match response {
4287                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4288                response => return worker_poll_response(response),
4289            }
4290        }
4291    }
4292
4293    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4294        match protocol {
4295            RequestProtocol::Worker(_) => {
4296                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4297                    return Ok(Some(token));
4298                }
4299                if self.control_token.is_some() {
4300                    return Err(Error::MissingRoleCredentials {
4301                        role: "worker",
4302                        opposite_role: "control",
4303                    });
4304                }
4305                Ok(None)
4306            }
4307            RequestProtocol::ControlPlane => {
4308                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4309                    return Ok(Some(token));
4310                }
4311                if self.worker_token.is_some() {
4312                    return Err(Error::MissingRoleCredentials {
4313                        role: "control",
4314                        opposite_role: "worker",
4315                    });
4316                }
4317                Ok(None)
4318            }
4319        }
4320    }
4321}
4322
4323fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4324    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4325    let reason = body
4326        .get("reason")
4327        .and_then(Value::as_str)
4328        .unwrap_or("query_rejected")
4329        .to_string();
4330    let message = body
4331        .get("message")
4332        .or_else(|| body.get("error"))
4333        .and_then(Value::as_str)
4334        .unwrap_or("workflow query was rejected")
4335        .to_string();
4336
4337    QueryFailure {
4338        status: status.as_u16(),
4339        reason,
4340        message,
4341        body,
4342    }
4343}
4344
4345fn workflow_command_result(
4346    command: WorkflowCommandKind,
4347    data: Value,
4348    workflow_id: &str,
4349    run_id: Option<&str>,
4350) -> WorkflowCommandResult {
4351    WorkflowCommandResult {
4352        command,
4353        workflow_id: data
4354            .get("workflow_id")
4355            .and_then(Value::as_str)
4356            .unwrap_or(workflow_id)
4357            .to_string(),
4358        run_id: data
4359            .get("run_id")
4360            .and_then(Value::as_str)
4361            .or(run_id)
4362            .map(str::to_string),
4363        outcome: data
4364            .get("outcome")
4365            .and_then(Value::as_str)
4366            .map(str::to_string),
4367        reason: data
4368            .get("reason")
4369            .and_then(Value::as_str)
4370            .map(str::to_string),
4371        command_status: data
4372            .get("command_status")
4373            .and_then(Value::as_str)
4374            .map(str::to_string),
4375        raw: data,
4376    }
4377}
4378
4379fn workflow_command_rejection(
4380    command: WorkflowCommandKind,
4381    status: reqwest::StatusCode,
4382    raw_body: String,
4383    workflow_id: &str,
4384    run_id: Option<&str>,
4385) -> WorkflowCommandRejection {
4386    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4387    WorkflowCommandRejection {
4388        command,
4389        status: status.as_u16(),
4390        reason: body
4391            .get("reason")
4392            .and_then(Value::as_str)
4393            .unwrap_or("workflow_command_rejected")
4394            .to_string(),
4395        message: body
4396            .get("message")
4397            .or_else(|| body.get("error"))
4398            .and_then(Value::as_str)
4399            .unwrap_or("workflow lifecycle command was rejected")
4400            .to_string(),
4401        workflow_id: body
4402            .get("workflow_id")
4403            .and_then(Value::as_str)
4404            .unwrap_or(workflow_id)
4405            .to_string(),
4406        run_id: body
4407            .get("run_id")
4408            .and_then(Value::as_str)
4409            .or(run_id)
4410            .map(str::to_string),
4411        target_scope: body
4412            .get("target_scope")
4413            .and_then(Value::as_str)
4414            .map(str::to_string),
4415        body,
4416    }
4417}
4418
4419fn query_task_response(response: Result<Value>) -> Result<Value> {
4420    match response {
4421        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4422        response => response,
4423    }
4424}
4425
4426fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4427    match response {
4428        Err(Error::Http { status, body })
4429            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4430        {
4431            Ok(serde_json::from_str(&body)?)
4432        }
4433        response => response,
4434    }
4435}
4436
4437fn worker_poll_body_is_stop(body: &str) -> bool {
4438    serde_json::from_str::<Value>(body)
4439        .ok()
4440        .is_some_and(|body| {
4441            worker_poll_is_stop(
4442                body.get("poll_status").and_then(Value::as_str),
4443                body.get("reason").and_then(Value::as_str),
4444            )
4445        })
4446}
4447
4448fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4449    matches!(poll_status, Some("draining" | "stopped"))
4450        || matches!(reason, Some("worker_draining" | "worker_stopped"))
4451}
4452
4453fn query_task_rejection_is_final(error: &Error) -> bool {
4454    matches!(
4455        error,
4456        Error::QueryFailed(failure)
4457            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4458    )
4459}
4460
4461fn activity_task_response<T>(
4462    response: Result<T>,
4463    operation: &str,
4464    task_id: &str,
4465    activity_attempt_id: &str,
4466) -> Result<T> {
4467    match response {
4468        Err(Error::Http { status, body }) => {
4469            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4470            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4471                operation: operation.to_string(),
4472                status: status.as_u16(),
4473                reason: body
4474                    .get("reason")
4475                    .and_then(Value::as_str)
4476                    .unwrap_or("activity_task_rejected")
4477                    .to_string(),
4478                task_id: body
4479                    .get("task_id")
4480                    .and_then(Value::as_str)
4481                    .unwrap_or(task_id)
4482                    .to_string(),
4483                activity_attempt_id: body
4484                    .get("activity_attempt_id")
4485                    .and_then(Value::as_str)
4486                    .unwrap_or(activity_attempt_id)
4487                    .to_string(),
4488                cancel_requested: body
4489                    .get("cancel_requested")
4490                    .and_then(Value::as_bool)
4491                    .unwrap_or(false),
4492                can_continue: body.get("can_continue").and_then(Value::as_bool),
4493                run_closed_reason: body
4494                    .get("run_closed_reason")
4495                    .and_then(Value::as_str)
4496                    .map(str::to_string),
4497                body,
4498            }))
4499        }
4500        response => response,
4501    }
4502}
4503
4504fn activity_task_rejection_is_final(error: &Error) -> bool {
4505    matches!(
4506        error,
4507        Error::ActivityTaskRejected(rejection)
4508            if matches!(
4509                rejection.reason.as_str(),
4510                "run_cancelled"
4511                    | "run_terminated"
4512                    | "attempt_closed"
4513                    | "stale_attempt"
4514                    | "activity_cancelled"
4515                    | "task_cancelled"
4516                    | "run_closed"
4517                    | "activity_not_running"
4518                    | "attempt_not_found"
4519            )
4520    )
4521}
4522
4523fn workflow_task_completion_is_terminal_timeout(
4524    error: &Error,
4525    task_id: &str,
4526    workflow_task_attempt: u64,
4527    run_id: Option<&str>,
4528) -> bool {
4529    let Error::Http { status, body } = error else {
4530        return false;
4531    };
4532    if *status != reqwest::StatusCode::CONFLICT {
4533        return false;
4534    }
4535
4536    let Some(run_id) = run_id else {
4537        return false;
4538    };
4539    let Ok(body) = serde_json::from_str::<Value>(body) else {
4540        return false;
4541    };
4542
4543    body.get("recorded").and_then(Value::as_bool) == Some(false)
4544        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4545        && body.get("run_status").and_then(Value::as_str) == Some("failed")
4546        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4547        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4548        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4549}
4550
4551fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4552    let body: Value = serde_json::from_str(raw_body).ok()?;
4553    let reason = body.get("reason")?.as_str()?;
4554    if !matches!(
4555        reason,
4556        "missing_protocol_version"
4557            | "unsupported_protocol_version"
4558            | "missing_control_plane_version"
4559            | "unsupported_control_plane_version"
4560    ) {
4561        return None;
4562    }
4563
4564    Some(ProtocolFailure {
4565        status: status.as_u16(),
4566        reason: reason.to_string(),
4567        message: body
4568            .get("message")
4569            .or_else(|| body.get("error"))
4570            .and_then(Value::as_str)
4571            .unwrap_or("protocol version rejected")
4572            .to_string(),
4573        supported_version: body
4574            .get("supported_version")
4575            .and_then(Value::as_str)
4576            .map(str::to_string),
4577        requested_version: body
4578            .get("requested_version")
4579            .and_then(Value::as_str)
4580            .map(str::to_string),
4581        body,
4582    })
4583}
4584
4585fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4586    timeout
4587        .as_secs()
4588        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4589        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4590}
4591
4592fn worker_operation_is_retryable(error: &Error) -> bool {
4593    if worker_poll_capacity_retry_after(error).is_some()
4594        || worker_storage_admission_body(error).is_some()
4595        || worker_backend_unavailable_body(error).is_some()
4596        || worker_operation_is_explicitly_non_retryable(error)
4597    {
4598        return false;
4599    }
4600
4601    match error {
4602        Error::Transport(error) => {
4603            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4604        }
4605        Error::Http { status, .. } => {
4606            matches!(
4607                *status,
4608                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4609            ) || status.is_server_error()
4610        }
4611        _ => false,
4612    }
4613}
4614
4615fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4616    let body: Value = match error {
4617        Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4618        Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4619        _ => return None,
4620    };
4621    matches!(
4622        body.get("reason").and_then(Value::as_str),
4623        Some("storage_pressure" | "storage_admission_unavailable")
4624    )
4625    .then_some(body)
4626}
4627
4628fn worker_storage_admission_retry_after(
4629    error: &Error,
4630    poll_request_id: Option<&str>,
4631) -> Option<Duration> {
4632    let Error::Http { status, .. } = error else {
4633        return None;
4634    };
4635    let body = worker_storage_admission_body(error)?;
4636    let delay = body.get("retry_after_seconds")?.as_u64()?;
4637    if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4638        || delay == 0
4639        || body.get("retryable") != Some(&Value::Bool(true))
4640        || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4641        || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4642        || body
4643            .get("request_admitted")
4644            .is_some_and(|admitted| admitted != &Value::Bool(false))
4645    {
4646        return None;
4647    }
4648    match poll_request_id {
4649        Some(id) => {
4650            if id.is_empty()
4651                || body.get("task") != Some(&Value::Null)
4652                || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4653                || body.get("poll_status") != body.get("reason")
4654                || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4655                || body.get("claim_admitted") != Some(&Value::Bool(false))
4656            {
4657                return None;
4658            }
4659        }
4660        None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4661        None => {}
4662    }
4663    Some(Duration::from_secs(delay))
4664}
4665
4666fn worker_backend_unavailable_retry_after(
4667    error: &Error,
4668    path: &str,
4669    poll_request_id: Option<&str>,
4670) -> Option<Duration> {
4671    let operation = match path {
4672        "/worker/register" => "register_worker",
4673        "/worker/heartbeat" => "heartbeat_worker",
4674        "/worker/workflow-tasks/poll" => "poll_workflow_task",
4675        "/worker/activity-tasks/poll" => "poll_activity_task",
4676        "/worker/query-tasks/poll" => "poll_query_task",
4677        "/worker/update-validation-tasks/poll" => "poll_update_validation_task",
4678        _ => return None,
4679    };
4680    let body = worker_backend_unavailable_body(error)?;
4681    let delay = body.get("retry_after_seconds")?.as_u64()?;
4682    if delay == 0
4683        || body.get("operation")?.as_str()? != operation
4684        || body.get("outcome")?.as_str()? != "unknown"
4685        || body.get("retryable") != Some(&Value::Bool(true))
4686        || body.get("worker_id")?.as_str()?.is_empty()
4687    {
4688        return None;
4689    }
4690    if let Some(id) = poll_request_id {
4691        if id.is_empty()
4692            || body.get("task") != Some(&Value::Null)
4693            || body.get("poll_status")?.as_str()? != "backend_unavailable"
4694            || body.get("poll_request_id")?.as_str()? != id
4695            || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4696        {
4697            return None;
4698        }
4699    } else if path.ends_with("/poll") {
4700        return None;
4701    }
4702    Some(Duration::from_secs(delay))
4703}
4704
4705fn worker_backend_unavailable_body(error: &Error) -> Option<Value> {
4706    let Error::Http { status, body } = error else {
4707        return None;
4708    };
4709    if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE {
4710        return None;
4711    }
4712    let body: Value = serde_json::from_str(body).ok()?;
4713    (body.get("reason")?.as_str()? == "backend_unavailable").then_some(body)
4714}
4715
4716fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4717    let Error::Http { body, .. } = error else {
4718        return false;
4719    };
4720
4721    serde_json::from_str::<Value>(body)
4722        .ok()
4723        .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4724        == Some(false)
4725}
4726
4727fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4728    let Error::Http { status, body } = error else {
4729        return None;
4730    };
4731    if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4732        return None;
4733    }
4734
4735    let body = serde_json::from_str::<Value>(body).ok()?;
4736    let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4737        == Some("long_poll_capacity_exhausted")
4738        || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4739    if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4740        return None;
4741    }
4742
4743    Some(Duration::from_secs(
4744        body.get("retry_after_seconds")
4745            .and_then(Value::as_u64)
4746            .unwrap_or_default(),
4747    ))
4748}
4749
4750fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4751    let exponent = retry.saturating_sub(1).min(31) as u32;
4752    policy
4753        .initial_backoff
4754        .saturating_mul(1_u32 << exponent)
4755        .min(policy.max_backoff)
4756}
4757
4758#[derive(Debug)]
4759pub struct ClientBuilder {
4760    base_url: String,
4761    token: Option<String>,
4762    control_token: Option<String>,
4763    worker_token: Option<String>,
4764    namespace: String,
4765    timeout: Duration,
4766    max_external_payload_bytes: usize,
4767}
4768
4769impl ClientBuilder {
4770    pub fn token(mut self, token: Option<String>) -> Self {
4771        self.token = token;
4772        self
4773    }
4774
4775    pub fn control_token(mut self, token: Option<String>) -> Self {
4776        self.control_token = token;
4777        self
4778    }
4779
4780    pub fn worker_token(mut self, token: Option<String>) -> Self {
4781        self.worker_token = token;
4782        self
4783    }
4784
4785    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4786        self.namespace = namespace.into();
4787        self
4788    }
4789
4790    pub fn timeout(mut self, timeout: Duration) -> Self {
4791        self.timeout = timeout;
4792        self
4793    }
4794
4795    /// Maximum unique external-payload bytes fetched per response (default 64 MiB).
4796    /// References are fetched only from this client's authenticated runtime.
4797    pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
4798        self.max_external_payload_bytes = bytes;
4799        self
4800    }
4801
4802    pub fn build(self) -> Result<Client> {
4803        let base_url = self.base_url.trim_end_matches('/').to_string();
4804        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4805            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4806            .unwrap_or_else(|_| base_url.ends_with("/api"));
4807
4808        if has_sdk_api_suffix {
4809            return Err(Error::InvalidBaseUrl);
4810        }
4811
4812        Ok(Client {
4813            http: reqwest::Client::builder()
4814                .timeout(self.timeout)
4815                .redirect(reqwest::redirect::Policy::none())
4816                .build()?,
4817            base_url,
4818            token: self.token,
4819            control_token: self.control_token,
4820            worker_token: self.worker_token,
4821            namespace: self.namespace,
4822            max_external_payload_bytes: self.max_external_payload_bytes,
4823            worker_storage_admission: None,
4824            runtime_upload_policy: Arc::new(Mutex::new([None, None])),
4825        })
4826    }
4827}
4828
4829#[derive(Clone, Debug)]
4830pub struct WorkflowHandle {
4831    client: Client,
4832    pub workflow_id: String,
4833    pub run_id: Option<String>,
4834    pub workflow_type: String,
4835}
4836
4837impl WorkflowHandle {
4838    /// Describe whichever run is current for this stable workflow instance.
4839    pub async fn describe(&self) -> Result<WorkflowDescription> {
4840        self.client.describe_workflow(&self.workflow_id).await
4841    }
4842
4843    /// Describe the run identity originally selected by this handle.
4844    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4845        let run_id = self.run_id.as_deref().ok_or_else(|| {
4846            Error::Codec("run_id is required for selected-run description".to_string())
4847        })?;
4848        self.client
4849            .describe_workflow_run(&self.workflow_id, run_id)
4850            .await
4851    }
4852
4853    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4854        self.client
4855            .signal_workflow(&self.workflow_id, signal_name, input)
4856            .await
4857    }
4858
4859    pub async fn append_message<T: Serialize>(
4860        &self,
4861        stream_name: &str,
4862        message_id: &str,
4863        input: T,
4864    ) -> Result<Value> {
4865        self.client
4866            .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4867            .await
4868    }
4869
4870    /// Signal only if this handle's selected run is still current.
4871    pub async fn signal_selected_run<T: Serialize>(
4872        &self,
4873        signal_name: &str,
4874        input: T,
4875    ) -> Result<Value> {
4876        let run_id = self.run_id.as_deref().ok_or_else(|| {
4877            Error::Codec("run_id is required for selected-run signaling".to_string())
4878        })?;
4879        self.client
4880            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4881            .await
4882    }
4883
4884    /// Request cooperative cancellation of whichever run is current.
4885    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4886        self.client
4887            .cancel_workflow(&self.workflow_id, options)
4888            .await
4889    }
4890
4891    /// Request cancellation only if this handle's selected run is still current.
4892    pub async fn cancel_selected_run(
4893        &self,
4894        options: WorkflowCommandOptions,
4895    ) -> Result<WorkflowCommandResult> {
4896        let run_id = self.run_id.as_deref().ok_or_else(|| {
4897            Error::Codec("run_id is required for selected-run cancellation".to_string())
4898        })?;
4899        self.client
4900            .cancel_workflow_run(&self.workflow_id, run_id, options)
4901            .await
4902    }
4903
4904    /// Forcefully terminate whichever run is current.
4905    pub async fn terminate(
4906        &self,
4907        options: WorkflowCommandOptions,
4908    ) -> Result<WorkflowCommandResult> {
4909        self.client
4910            .terminate_workflow(&self.workflow_id, options)
4911            .await
4912    }
4913
4914    /// Terminate only if this handle's selected run is still current.
4915    pub async fn terminate_selected_run(
4916        &self,
4917        options: WorkflowCommandOptions,
4918    ) -> Result<WorkflowCommandResult> {
4919        let run_id = self.run_id.as_deref().ok_or_else(|| {
4920            Error::Codec("run_id is required for selected-run termination".to_string())
4921        })?;
4922        self.client
4923            .terminate_workflow_run(&self.workflow_id, run_id, options)
4924            .await
4925    }
4926
4927    /// Continue this handle's failed run without repeating completed activities.
4928    pub async fn redrive(&self, request_id: Option<&str>) -> Result<WorkflowRedriveResult> {
4929        let run_id = self
4930            .run_id
4931            .as_deref()
4932            .ok_or_else(|| Error::Codec("run_id is required for redrive".to_string()))?;
4933        self.client
4934            .redrive_workflow_run(&self.workflow_id, run_id, request_id)
4935            .await
4936    }
4937
4938    /// Execute a named, read-only query against this workflow.
4939    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4940        self.client
4941            .query_workflow(&self.workflow_id, query_name, input)
4942            .await
4943    }
4944
4945    pub async fn query_avro_value<T: Serialize>(
4946        &self,
4947        query_name: &str,
4948        input: T,
4949    ) -> Result<AvroValue> {
4950        self.client
4951            .query_workflow_avro_value(&self.workflow_id, query_name, input)
4952            .await
4953    }
4954
4955    pub async fn update<T: Serialize>(
4956        &self,
4957        update_name: &str,
4958        input: T,
4959        request_id: Option<&str>,
4960    ) -> Result<Value> {
4961        self.client
4962            .update_workflow(&self.workflow_id, update_name, input, request_id)
4963            .await
4964    }
4965
4966    pub async fn update_avro_value<T: Serialize>(
4967        &self,
4968        update_name: &str,
4969        input: T,
4970        request_id: Option<&str>,
4971    ) -> Result<AvroValue> {
4972        self.client
4973            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4974            .await
4975    }
4976
4977    /// Query only if this handle's selected run is still current.
4978    pub async fn query_selected_run<T: Serialize>(
4979        &self,
4980        query_name: &str,
4981        input: T,
4982    ) -> Result<Value> {
4983        let run_id = self
4984            .run_id
4985            .as_deref()
4986            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4987        self.client
4988            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4989            .await
4990    }
4991
4992    /// Await the final terminal outcome of the current continue-as-new chain.
4993    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4994        self.result_target(options, None).await
4995    }
4996
4997    /// Await the final result without projecting Avro bytes through JSON.
4998    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4999        self.result_avro_value_target(options, None).await
5000    }
5001
5002    /// Await the final result and decode it into a Serde application type.
5003    pub async fn result_typed<T: DeserializeOwned>(
5004        &self,
5005        options: WorkflowResultOptions,
5006    ) -> Result<T> {
5007        let result = self.result_avro_value(options).await?;
5008        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5009    }
5010
5011    /// Await only the run identity originally selected by this handle.
5012    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
5013        let run_id = self.run_id.as_deref().ok_or_else(|| {
5014            Error::Codec("run_id is required for selected-run result".to_string())
5015        })?;
5016        self.result_target(options, Some(run_id)).await
5017    }
5018
5019    /// Await the selected run's result on the lossless Avro Value surface.
5020    pub async fn result_selected_run_avro_value(
5021        &self,
5022        options: WorkflowResultOptions,
5023    ) -> Result<AvroValue> {
5024        let run_id = self.run_id.as_deref().ok_or_else(|| {
5025            Error::Codec("run_id is required for selected-run result".to_string())
5026        })?;
5027        self.result_avro_value_target(options, Some(run_id)).await
5028    }
5029
5030    /// Await the selected run and decode its result into a Serde type.
5031    pub async fn result_selected_run_typed<T: DeserializeOwned>(
5032        &self,
5033        options: WorkflowResultOptions,
5034    ) -> Result<T> {
5035        let result = self.result_selected_run_avro_value(options).await?;
5036        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5037    }
5038
5039    async fn result_avro_value_target(
5040        &self,
5041        options: WorkflowResultOptions,
5042        selected_run_id: Option<&str>,
5043    ) -> Result<AvroValue> {
5044        let started = Instant::now();
5045
5046        loop {
5047            let description = match selected_run_id {
5048                Some(run_id) => {
5049                    self.client
5050                        .describe_workflow_run(&self.workflow_id, run_id)
5051                        .await?
5052                }
5053                None => self.describe().await?,
5054            };
5055            if description.is_completed() {
5056                return description.output_avro_value.ok_or_else(|| {
5057                    Error::Codec(
5058                        "missing_payload_envelope: typed workflow result requires output_envelope"
5059                            .to_string(),
5060                    )
5061                });
5062            }
5063            if description.is_terminal() {
5064                let outcome =
5065                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5066                return Err(match outcome.kind {
5067                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5068                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5069                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5070                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5071                });
5072            }
5073            if started.elapsed() >= options.timeout {
5074                return Err(Error::Timeout);
5075            }
5076            tokio::time::sleep(options.poll_interval).await;
5077        }
5078    }
5079
5080    async fn result_target(
5081        &self,
5082        options: WorkflowResultOptions,
5083        selected_run_id: Option<&str>,
5084    ) -> Result<Value> {
5085        let started = Instant::now();
5086
5087        loop {
5088            let description = match selected_run_id {
5089                Some(run_id) => {
5090                    self.client
5091                        .describe_workflow_run(&self.workflow_id, run_id)
5092                        .await?
5093                }
5094                None => self.describe().await?,
5095            };
5096            if description.is_completed() {
5097                return Ok(description.output.unwrap_or(Value::Null));
5098            }
5099
5100            if description.is_terminal() {
5101                let outcome =
5102                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5103                return Err(match outcome.kind {
5104                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5105                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5106                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5107                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5108                });
5109            }
5110
5111            if started.elapsed() >= options.timeout {
5112                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
5113                    kind: WorkflowTerminalKind::TimedOut,
5114                    workflow_id: description
5115                        .workflow_id
5116                        .clone()
5117                        .unwrap_or_else(|| self.workflow_id.clone()),
5118                    run_id: description
5119                        .run_id
5120                        .clone()
5121                        .or_else(|| selected_run_id.map(str::to_string)),
5122                    reason: "result_wait_timeout".to_string(),
5123                    failure_category: Some("client_timeout".to_string()),
5124                    failure_id: None,
5125                    exception_type: None,
5126                    exception_class: None,
5127                    non_retryable: None,
5128                    message: Some(format!(
5129                        "workflow result was not terminal within {:?}",
5130                        options.timeout
5131                    )),
5132                    exception: None,
5133                    raw: description.raw_value(),
5134                }));
5135            }
5136
5137            tokio::time::sleep(options.poll_interval).await;
5138        }
5139    }
5140}
5141
5142#[derive(Clone, Copy, Debug)]
5143pub struct WorkflowResultOptions {
5144    pub poll_interval: Duration,
5145    pub timeout: Duration,
5146}
5147
5148impl Default for WorkflowResultOptions {
5149    fn default() -> Self {
5150        Self {
5151            poll_interval: Duration::from_millis(500),
5152            timeout: Duration::from_secs(30),
5153        }
5154    }
5155}
5156
5157#[derive(Clone, Debug, Deserialize)]
5158pub struct WorkflowDescription {
5159    pub workflow_id: Option<String>,
5160    pub run_id: Option<String>,
5161    pub workflow_type: Option<String>,
5162    pub status: Option<String>,
5163    #[serde(default)]
5164    pub closed_reason: Option<String>,
5165    #[serde(default)]
5166    pub error: Option<String>,
5167    #[serde(default)]
5168    pub failure: Option<Value>,
5169    #[serde(default)]
5170    pub exception: Option<Value>,
5171    #[serde(default)]
5172    pub failures: Vec<Value>,
5173    #[serde(default)]
5174    pub output: Option<Value>,
5175    #[serde(default)]
5176    pub output_envelope: Option<Value>,
5177    #[serde(skip)]
5178    pub output_avro_value: Option<AvroValue>,
5179    #[serde(flatten)]
5180    pub raw: HashMap<String, Value>,
5181}
5182
5183/// Lifecycle and backlog metadata for one run-scoped Workflow Stream.
5184#[derive(Clone, Debug, Deserialize)]
5185pub struct WorkflowStreamDescription {
5186    pub stream_name: String,
5187    pub status: String,
5188    pub last_offset: i64,
5189    pub total_items: u64,
5190    pub pending_items: u64,
5191    #[serde(default)]
5192    pub opened_at: Option<String>,
5193    #[serde(default)]
5194    pub last_appended_at: Option<String>,
5195    #[serde(default)]
5196    pub closed_at: Option<String>,
5197    #[serde(default)]
5198    pub error_reason: Option<String>,
5199    #[serde(default)]
5200    pub retention_seconds: Option<u64>,
5201    #[serde(flatten)]
5202    pub raw: HashMap<String, Value>,
5203}
5204
5205impl WorkflowStreamDescription {
5206    pub fn is_terminal(&self) -> bool {
5207        matches!(self.status.as_str(), "closed" | "errored")
5208    }
5209}
5210
5211/// One item for direct or replay-safe append.
5212#[derive(Clone, Debug, Default)]
5213pub struct WorkflowStreamAppendItem {
5214    pub payload_envelope: Option<Value>,
5215    pub payload_reference: Option<String>,
5216    pub item_type: Option<String>,
5217    pub content_type: Option<String>,
5218    pub idempotency_key: Option<String>,
5219}
5220
5221impl WorkflowStreamAppendItem {
5222    /// Encode an inline payload with the SDK's fixed Avro Value envelope.
5223    pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5224        let value = AvroValue::from_serialize(&payload)?;
5225        Ok(Self {
5226            payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5227            ..Self::default()
5228        })
5229    }
5230
5231    /// Preserve an external payload URI as an opaque service-contract reference.
5232    pub fn from_reference(reference: impl Into<String>) -> Self {
5233        Self {
5234            payload_reference: Some(reference.into()),
5235            ..Self::default()
5236        }
5237    }
5238
5239    pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5240        self.item_type = Some(item_type.into());
5241        self
5242    }
5243
5244    pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5245        self.content_type = Some(content_type.into());
5246        self
5247    }
5248
5249    pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5250        self.idempotency_key = Some(idempotency_key.into());
5251        self
5252    }
5253
5254    fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5255        let mut item = serde_json::Map::new();
5256        if let Some(payload) = &self.payload_envelope {
5257            item.insert("payload".to_string(), payload.clone());
5258            item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5259        }
5260        if let Some(reference) = &self.payload_reference {
5261            item.insert("payload_reference".to_string(), json!(reference));
5262        }
5263        if let Some(item_type) = &self.item_type {
5264            item.insert("item_type".to_string(), json!(item_type));
5265        }
5266        if let Some(content_type) = &self.content_type {
5267            item.insert("content_type".to_string(), json!(content_type));
5268        }
5269        if let Some(key) = derived_idempotency_key
5270            .as_ref()
5271            .or(self.idempotency_key.as_ref())
5272        {
5273            item.insert("idempotency_key".to_string(), json!(key));
5274        }
5275        Value::Object(item)
5276    }
5277}
5278
5279/// One durable item at its stable zero-based offset.
5280#[derive(Clone, Debug)]
5281pub struct WorkflowStreamItem {
5282    pub offset: u64,
5283    pub payload: Option<Value>,
5284    pub payload_envelope: Option<Value>,
5285    pub payload_reference: Option<String>,
5286    pub payload_codec: Option<String>,
5287    pub idempotency_key: Option<String>,
5288    pub item_type: Option<String>,
5289    pub content_type: Option<String>,
5290    pub origin: Option<String>,
5291    pub origin_reference: Option<String>,
5292    pub emitted_at: Option<String>,
5293    pub raw: Value,
5294}
5295
5296/// One bounded at-least-once subscription page.
5297#[derive(Clone, Debug)]
5298pub struct WorkflowStreamPage {
5299    pub stream: WorkflowStreamDescription,
5300    pub items: Vec<WorkflowStreamItem>,
5301    pub next_offset: u64,
5302    pub terminal: bool,
5303}
5304
5305/// Durable acceptance and deduplication outcome for an append request.
5306#[derive(Clone, Debug)]
5307pub struct WorkflowStreamAppendResult {
5308    pub stream: WorkflowStreamDescription,
5309    pub accepted_offsets: Vec<u64>,
5310    pub accepted: u64,
5311    pub deduped: u64,
5312}
5313
5314#[derive(Deserialize)]
5315struct WorkflowStreamListResponse {
5316    #[serde(default)]
5317    streams: Vec<WorkflowStreamDescription>,
5318}
5319
5320#[derive(Deserialize)]
5321struct WorkflowStreamDescriptionResponse {
5322    stream: WorkflowStreamDescription,
5323}
5324
5325#[derive(Deserialize)]
5326struct WorkflowStreamPageResponse {
5327    stream: WorkflowStreamDescription,
5328    #[serde(default)]
5329    items: Vec<Value>,
5330    next_offset: u64,
5331    terminal: bool,
5332}
5333
5334#[derive(Deserialize)]
5335struct WorkflowStreamAppendResponse {
5336    stream: WorkflowStreamDescription,
5337    #[serde(default)]
5338    accepted_offsets: Vec<u64>,
5339    accepted: u64,
5340    deduped: u64,
5341}
5342
5343impl WorkflowDescription {
5344    pub fn is_completed(&self) -> bool {
5345        matches!(self.status.as_deref(), Some("completed" | "Completed"))
5346    }
5347
5348    pub fn is_terminal(&self) -> bool {
5349        matches!(
5350            self.status.as_deref(),
5351            Some(
5352                "completed"
5353                    | "Completed"
5354                    | "failed"
5355                    | "Failed"
5356                    | "cancelled"
5357                    | "Cancelled"
5358                    | "terminated"
5359                    | "Terminated"
5360                    | "timed_out"
5361                    | "TimedOut",
5362            )
5363        )
5364    }
5365
5366    fn decode_payloads(&mut self) -> Result<()> {
5367        if let Some(envelope) = &self.output_envelope {
5368            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5369            self.output = Some(value.clone().into_json()?);
5370            self.output_avro_value = Some(value);
5371        }
5372
5373        Ok(())
5374    }
5375
5376    fn raw_value(&self) -> Value {
5377        let mut data = self.raw.clone();
5378        data.insert(
5379            "workflow_id".to_string(),
5380            self.workflow_id
5381                .clone()
5382                .map(Value::String)
5383                .unwrap_or(Value::Null),
5384        );
5385        data.insert(
5386            "run_id".to_string(),
5387            self.run_id
5388                .clone()
5389                .map(Value::String)
5390                .unwrap_or(Value::Null),
5391        );
5392        data.insert(
5393            "workflow_type".to_string(),
5394            self.workflow_type
5395                .clone()
5396                .map(Value::String)
5397                .unwrap_or(Value::Null),
5398        );
5399        data.insert(
5400            "status".to_string(),
5401            self.status
5402                .clone()
5403                .map(Value::String)
5404                .unwrap_or(Value::Null),
5405        );
5406        data.insert(
5407            "closed_reason".to_string(),
5408            self.closed_reason
5409                .clone()
5410                .map(Value::String)
5411                .unwrap_or(Value::Null),
5412        );
5413        if let Some(failure) = &self.failure {
5414            data.insert("failure".to_string(), failure.clone());
5415        }
5416        if let Some(exception) = &self.exception {
5417            data.insert("exception".to_string(), exception.clone());
5418        }
5419        Value::Object(data.into_iter().collect())
5420    }
5421}
5422
5423fn workflow_terminal_outcome(
5424    description: &WorkflowDescription,
5425    workflow_id: &str,
5426    run_id: Option<&str>,
5427) -> WorkflowTerminalOutcome {
5428    let terminal_kind = description
5429        .closed_reason
5430        .as_deref()
5431        .or(description.status.as_deref())
5432        .unwrap_or("failed")
5433        .to_ascii_lowercase();
5434    let kind = match terminal_kind.as_str() {
5435        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5436        "terminated" => WorkflowTerminalKind::Terminated,
5437        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5438        _ => WorkflowTerminalKind::Failed,
5439    };
5440    let default_reason = match kind {
5441        WorkflowTerminalKind::Failed => "workflow_failed",
5442        WorkflowTerminalKind::Cancelled => "cancelled",
5443        WorkflowTerminalKind::Terminated => "terminated",
5444        WorkflowTerminalKind::TimedOut => "timed_out",
5445    };
5446    let failure = description
5447        .failure
5448        .as_ref()
5449        .filter(|value| value.is_object());
5450    let nested_failure = failure
5451        .and_then(|value| value.get("failures"))
5452        .and_then(Value::as_array)
5453        .and_then(|failures| failures.last())
5454        .or_else(|| description.failures.last());
5455    let exception = description
5456        .exception
5457        .clone()
5458        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5459        .or_else(|| {
5460            nested_failure
5461                .and_then(|value| value.get("exception_payload"))
5462                .cloned()
5463        });
5464    let string_field = |name: &str| {
5465        failure
5466            .and_then(|value| value.get(name))
5467            .and_then(Value::as_str)
5468            .or_else(|| {
5469                nested_failure
5470                    .and_then(|value| value.get(name))
5471                    .and_then(Value::as_str)
5472            })
5473            .map(str::to_string)
5474    };
5475    let exception_field = |name: &str| {
5476        exception
5477            .as_ref()
5478            .and_then(|value| value.get(name))
5479            .and_then(Value::as_str)
5480            .map(str::to_string)
5481    };
5482    let message = description
5483        .error
5484        .clone()
5485        .or_else(|| string_field("message"))
5486        .or_else(|| exception_field("message"));
5487    let reason = description
5488        .raw
5489        .get("reason")
5490        .and_then(Value::as_str)
5491        .map(str::to_string)
5492        .or_else(|| {
5493            failure
5494                .and_then(|value| value.get("reason"))
5495                .and_then(Value::as_str)
5496                .map(str::to_string)
5497        })
5498        .or_else(|| description.closed_reason.clone())
5499        .unwrap_or_else(|| default_reason.to_string());
5500    let failure_id = string_field("failure_id").or_else(|| {
5501        nested_failure
5502            .and_then(|value| value.get("id"))
5503            .and_then(Value::as_str)
5504            .map(str::to_string)
5505    });
5506
5507    WorkflowTerminalOutcome {
5508        kind,
5509        workflow_id: description
5510            .workflow_id
5511            .clone()
5512            .unwrap_or_else(|| workflow_id.to_string()),
5513        run_id: description
5514            .run_id
5515            .clone()
5516            .or_else(|| run_id.map(str::to_string)),
5517        reason,
5518        failure_category: string_field("failure_category")
5519            .or_else(|| Some(default_reason.to_string())),
5520        failure_id,
5521        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5522        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5523        non_retryable: failure
5524            .and_then(|value| value.get("non_retryable"))
5525            .and_then(Value::as_bool)
5526            .or_else(|| {
5527                nested_failure
5528                    .and_then(|value| value.get("non_retryable"))
5529                    .and_then(Value::as_bool)
5530            }),
5531        message,
5532        exception,
5533        raw: description.raw_value(),
5534    }
5535}
5536
5537#[derive(Clone, Debug, Deserialize)]
5538pub struct RegisterWorkerResponse {
5539    pub worker_id: String,
5540    pub registered: bool,
5541    #[serde(default)]
5542    pub heartbeat_interval_seconds: Option<u64>,
5543    #[serde(default)]
5544    pub protocol_version: Option<String>,
5545    #[serde(default)]
5546    pub server_capabilities: Option<Value>,
5547}
5548
5549/// Result of gracefully removing a worker-plane registration.
5550#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5551pub struct WorkerDeregistrationEnvelope {
5552    pub worker_id: String,
5553    pub outcome: String,
5554    pub recovered_workflow_task_count: u64,
5555}
5556
5557#[derive(Clone, Debug, Deserialize)]
5558pub struct PollWorkflowTaskResponse {
5559    #[serde(default)]
5560    pub task: Option<WorkflowTask>,
5561    #[serde(default)]
5562    pub poll_status: Option<String>,
5563    #[serde(default)]
5564    pub reason: Option<String>,
5565    #[serde(default)]
5566    pub protocol_version: Option<String>,
5567    #[serde(default)]
5568    pub server_capabilities: Option<Value>,
5569}
5570
5571impl PollWorkflowTaskResponse {
5572    /// Classify this response without parsing server display text.
5573    pub fn outcome(&self) -> WorkerPollOutcome {
5574        worker_poll_outcome(
5575            self.task.is_some(),
5576            self.poll_status.as_deref(),
5577            self.reason.as_deref(),
5578        )
5579    }
5580}
5581
5582fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5583    let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5584        return false;
5585    };
5586    let supported = capabilities
5587        .get("workflow_memo_updates")
5588        .and_then(Value::as_object)
5589        .and_then(|memo| memo.get("supported"))
5590        .and_then(Value::as_bool)
5591        == Some(true);
5592    let command_advertised = capabilities
5593        .get("supported_workflow_task_commands")
5594        .and_then(Value::as_array)
5595        .is_some_and(|commands| {
5596            commands
5597                .iter()
5598                .any(|command| command.as_str() == Some("upsert_memo"))
5599        });
5600    supported && command_advertised
5601}
5602
5603fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5604    commands
5605        .iter()
5606        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5607}
5608
5609#[derive(Clone, Debug, Deserialize)]
5610pub struct PollActivityTaskResponse {
5611    #[serde(default)]
5612    pub task: Option<ActivityTask>,
5613    #[serde(default)]
5614    pub poll_status: Option<String>,
5615    #[serde(default)]
5616    pub reason: Option<String>,
5617}
5618
5619impl PollActivityTaskResponse {
5620    /// Classify this response without parsing server display text.
5621    pub fn outcome(&self) -> WorkerPollOutcome {
5622        worker_poll_outcome(
5623            self.task.is_some(),
5624            self.poll_status.as_deref(),
5625            self.reason.as_deref(),
5626        )
5627    }
5628}
5629
5630#[derive(Clone, Debug, Deserialize)]
5631pub struct PollQueryTaskResponse {
5632    #[serde(default)]
5633    pub task: Option<QueryTask>,
5634    #[serde(default)]
5635    pub poll_status: Option<String>,
5636    #[serde(default)]
5637    pub reason: Option<String>,
5638}
5639
5640impl PollQueryTaskResponse {
5641    /// Classify this response without parsing server display text.
5642    pub fn outcome(&self) -> WorkerPollOutcome {
5643        worker_poll_outcome(
5644            self.task.is_some(),
5645            self.poll_status.as_deref(),
5646            self.reason.as_deref(),
5647        )
5648    }
5649}
5650
5651/// Stable classification for worker poll responses.
5652#[derive(Clone, Debug, PartialEq, Eq)]
5653pub enum WorkerPollOutcome {
5654    /// A task was leased and is available on the response.
5655    Task,
5656    /// No task was leased, but the worker should continue polling.
5657    Idle {
5658        poll_status: Option<String>,
5659        reason: Option<String>,
5660    },
5661    /// The server asked this worker to stop claiming new work.
5662    Stop {
5663        poll_status: Option<String>,
5664        reason: Option<String>,
5665    },
5666}
5667
5668impl WorkerPollOutcome {
5669    pub fn should_stop(&self) -> bool {
5670        matches!(self, Self::Stop { .. })
5671    }
5672}
5673
5674fn worker_poll_outcome(
5675    has_task: bool,
5676    poll_status: Option<&str>,
5677    reason: Option<&str>,
5678) -> WorkerPollOutcome {
5679    if worker_poll_is_stop(poll_status, reason) {
5680        return WorkerPollOutcome::Stop {
5681            poll_status: poll_status.map(str::to_string),
5682            reason: reason.map(str::to_string),
5683        };
5684    }
5685
5686    if has_task {
5687        WorkerPollOutcome::Task
5688    } else {
5689        WorkerPollOutcome::Idle {
5690            poll_status: poll_status.map(str::to_string),
5691            reason: reason.map(str::to_string),
5692        }
5693    }
5694}
5695
5696/// An ephemeral server-routed query task.
5697#[derive(Clone, Debug, Deserialize)]
5698pub struct QueryTask {
5699    pub query_task_id: String,
5700    #[serde(default = "default_workflow_task_attempt")]
5701    pub query_task_attempt: u64,
5702    #[serde(default)]
5703    pub lease_owner: Option<String>,
5704    #[serde(default)]
5705    pub workflow_id: Option<String>,
5706    #[serde(default)]
5707    pub run_id: Option<String>,
5708    pub workflow_type: String,
5709    pub query_name: String,
5710    #[serde(
5711        default = "missing_task_payload_codec",
5712        deserialize_with = "deserialize_task_payload_codec"
5713    )]
5714    pub payload_codec: String,
5715    #[serde(default)]
5716    pub workflow_arguments: Option<Value>,
5717    #[serde(default)]
5718    pub query_arguments: Option<Value>,
5719    #[serde(default)]
5720    pub history_events: Vec<HistoryEvent>,
5721    #[serde(default)]
5722    pub history_export: Option<Value>,
5723    #[serde(default)]
5724    pub run_status: Option<String>,
5725}
5726
5727#[derive(Clone, Debug, Deserialize)]
5728pub struct WorkflowTask {
5729    pub task_id: String,
5730    #[serde(default)]
5731    pub workflow_command_id: Option<String>,
5732    #[serde(default)]
5733    pub workflow_id: Option<String>,
5734    #[serde(default)]
5735    pub run_id: Option<String>,
5736    pub workflow_type: String,
5737    #[serde(default)]
5738    pub cancel_requested: bool,
5739    #[serde(
5740        default = "missing_task_payload_codec",
5741        deserialize_with = "deserialize_task_payload_codec"
5742    )]
5743    pub payload_codec: String,
5744    #[serde(default)]
5745    pub arguments: Option<Value>,
5746    #[serde(default)]
5747    pub history_events: Vec<HistoryEvent>,
5748    #[serde(default)]
5749    pub total_history_events: Option<u64>,
5750    #[serde(default)]
5751    pub history_size_bytes: Option<u64>,
5752    #[serde(default)]
5753    pub continue_as_new_recommended: Option<bool>,
5754    #[serde(default)]
5755    pub history_budget_pressure: Option<String>,
5756    #[serde(default)]
5757    pub next_history_page_token: Option<String>,
5758    #[serde(default = "default_workflow_task_attempt")]
5759    pub workflow_task_attempt: u64,
5760    #[serde(default)]
5761    pub workflow_signal_id: Option<String>,
5762    #[serde(default)]
5763    pub signal_name: Option<String>,
5764    #[serde(default)]
5765    pub signal_arguments: Option<Value>,
5766    #[serde(default)]
5767    pub workflow_update_id: Option<String>,
5768    #[serde(default)]
5769    pub update_name: Option<String>,
5770    #[serde(default)]
5771    pub lease_owner: Option<String>,
5772}
5773
5774impl WorkflowTask {
5775    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5776        self.history_events.extend(page.history_events);
5777
5778        if page.total_history_events.is_some() {
5779            self.total_history_events = page.total_history_events;
5780        }
5781
5782        self.next_history_page_token = page
5783            .next_history_page_token
5784            .filter(|token| !token.is_empty());
5785    }
5786}
5787
5788#[derive(Clone, Debug, Deserialize)]
5789struct WorkflowTaskHistoryPage {
5790    #[serde(default)]
5791    history_events: Vec<HistoryEvent>,
5792    #[serde(default)]
5793    total_history_events: Option<u64>,
5794    #[serde(default)]
5795    next_history_page_token: Option<String>,
5796}
5797
5798#[derive(Clone, Debug, Deserialize)]
5799pub struct ActivityTask {
5800    pub task_id: String,
5801    #[serde(default)]
5802    pub activity_attempt_id: Option<String>,
5803    #[serde(default)]
5804    pub attempt_id: Option<String>,
5805    pub activity_type: String,
5806    #[serde(
5807        default = "missing_task_payload_codec",
5808        deserialize_with = "deserialize_task_payload_codec"
5809    )]
5810    pub payload_codec: String,
5811    #[serde(default)]
5812    pub arguments: Option<Value>,
5813    #[serde(default = "default_attempt_number")]
5814    pub attempt_number: u64,
5815    #[serde(default)]
5816    pub lease_owner: Option<String>,
5817}
5818
5819#[derive(Clone, Debug, Deserialize)]
5820pub struct HistoryEvent {
5821    #[serde(alias = "type")]
5822    pub event_type: String,
5823    #[serde(default)]
5824    pub payload: Value,
5825    #[serde(flatten)]
5826    pub raw: HashMap<String, Value>,
5827}
5828
5829/// One decoded signal in the committed workflow-history snapshot.
5830#[derive(Clone, Debug, PartialEq)]
5831pub struct QuerySignal {
5832    pub id: Option<String>,
5833    pub name: String,
5834    pub arguments: Vec<Value>,
5835    avro_arguments: Vec<AvroValue>,
5836    pub workflow_sequence: Option<u64>,
5837}
5838
5839impl QuerySignal {
5840    /// Lossless fixed Avro Value arguments for this committed signal.
5841    pub fn arguments_avro_value(&self) -> &[AvroValue] {
5842        &self.avro_arguments
5843    }
5844}
5845
5846/// Immutable state supplied to a registered query handler.
5847///
5848/// This context intentionally exposes no activity, signal-wait, or command
5849/// APIs. Query handlers inspect committed history and return a value; query
5850/// completion does not append an event or advance deterministic execution.
5851#[derive(Clone, Debug)]
5852pub struct QueryContext {
5853    pub workflow_id: Option<String>,
5854    pub run_id: Option<String>,
5855    pub workflow_type: String,
5856    pub run_status: Option<String>,
5857    workflow_input: Value,
5858    workflow_input_avro_value: AvroValue,
5859    history_events: Arc<Vec<HistoryEvent>>,
5860    signal_events: Arc<Vec<QuerySignal>>,
5861}
5862
5863impl QueryContext {
5864    /// The normalized argument list used to start the workflow.
5865    pub fn workflow_input(&self) -> &Value {
5866        &self.workflow_input
5867    }
5868
5869    /// The lossless fixed Avro Value argument list used to start the workflow.
5870    pub fn workflow_input_avro_value(&self) -> &AvroValue {
5871        &self.workflow_input_avro_value
5872    }
5873
5874    /// The immutable committed history used for this query snapshot.
5875    pub fn history_events(&self) -> &[HistoryEvent] {
5876        self.history_events.as_slice()
5877    }
5878
5879    /// All decoded signals in committed workflow order.
5880    pub fn signal_events(&self) -> &[QuerySignal] {
5881        self.signal_events.as_slice()
5882    }
5883
5884    /// Decoded argument lists for each committed signal with `signal_name`.
5885    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5886        self.signal_events
5887            .iter()
5888            .filter(|signal| signal.name == signal_name)
5889            .map(|signal| signal.arguments.clone())
5890            .collect()
5891    }
5892
5893    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
5894    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5895        self.signal_events
5896            .iter()
5897            .filter(|signal| signal.name == signal_name)
5898            .map(|signal| signal.avro_arguments.clone())
5899            .collect()
5900    }
5901}
5902
5903#[derive(Clone, Debug, Deserialize)]
5904pub struct ActivityHeartbeatResponse {
5905    #[serde(default)]
5906    pub cancel_requested: bool,
5907    #[serde(default)]
5908    pub heartbeat_recorded: bool,
5909    #[serde(default)]
5910    pub can_continue: Option<bool>,
5911    #[serde(default)]
5912    pub reason: Option<String>,
5913    #[serde(default)]
5914    pub run_closed_reason: Option<String>,
5915    #[serde(default)]
5916    pub run_closed_at: Option<String>,
5917    #[serde(default)]
5918    pub lease_expires_at: Option<String>,
5919    #[serde(default)]
5920    pub last_heartbeat_at: Option<String>,
5921}
5922
5923impl ActivityHeartbeatResponse {
5924    /// Whether the activity should stop instead of attempting completion.
5925    pub fn should_stop(&self) -> bool {
5926        self.cancel_requested || self.can_continue == Some(false)
5927    }
5928}
5929
5930fn missing_task_payload_codec() -> String {
5931    MISSING_TASK_PAYLOAD_CODEC.to_string()
5932}
5933
5934fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5935where
5936    D: Deserializer<'de>,
5937{
5938    Ok(match Value::deserialize(deserializer)? {
5939        Value::String(codec) => codec,
5940        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5941        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5942    })
5943}
5944
5945fn default_workflow_task_attempt() -> u64 {
5946    1
5947}
5948
5949fn default_attempt_number() -> u64 {
5950    1
5951}
5952
5953type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5954type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5955type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5956type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5957type ReplayedWorkflowHandler =
5958    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5959type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5960type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5961type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5962type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5963type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5964type ReplayedQueryHandler = Arc<
5965    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5966        + Send
5967        + Sync,
5968>;
5969type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5970
5971struct ReplayedWorkflowInvocation {
5972    future: WorkflowFuture,
5973    snapshot: WorkflowStateSnapshot,
5974}
5975
5976#[derive(Clone)]
5977struct RegisteredWorkflow {
5978    execute: WorkflowHandler,
5979    replay: Option<ReplayedWorkflowHandler>,
5980    state_type: Option<TypeId>,
5981    definition_fingerprint: Option<String>,
5982}
5983
5984#[derive(Debug)]
5985struct WorkflowTaskDecision {
5986    commands: Vec<Value>,
5987    message_stream_cursors: Vec<Value>,
5988    message_stream_waits: Vec<Value>,
5989}
5990
5991impl WorkflowTaskDecision {
5992    fn without_message_streams(commands: Vec<Value>) -> Self {
5993        Self {
5994            commands,
5995            message_stream_cursors: Vec::new(),
5996            message_stream_waits: Vec::new(),
5997        }
5998    }
5999}
6000
6001#[derive(Clone)]
6002enum RegisteredQuery {
6003    Snapshot(QueryHandler),
6004    Replayed {
6005        state_type: TypeId,
6006        handler: ReplayedQueryHandler,
6007    },
6008}
6009
6010#[derive(Clone, Debug)]
6011pub struct WorkerHeartbeatObservation {
6012    pub worker_id: String,
6013    pub task_queue: String,
6014    pub acknowledged_at_unix_millis: u64,
6015    pub acknowledgement: Value,
6016}
6017
6018/// Bounded retry policy for worker poll acquisition and worker heartbeats.
6019///
6020/// Expected empty long polls and explicit Server capacity backpressure are
6021/// normal worker states. Capacity backpressure honors the advertised retry
6022/// delay without consuming this retry budget. Other transport failures,
6023/// retryable HTTP 408/429 responses, and server errors are retried with capped
6024/// exponential backoff. Authentication, protocol, codec, and handler failures
6025/// are never retried by the worker.
6026/// Explicit storage admission pauses also preserve the already-serialized worker
6027/// request without consuming this budget or rerunning its handler. The delay is
6028/// capped by `max_backoff`, and shutdown interrupts storage waits.
6029#[derive(Clone, Copy, Debug)]
6030pub struct WorkerRetryPolicy {
6031    /// Number of retries after the initial request fails.
6032    pub max_retries: usize,
6033    /// Delay before the first retry.
6034    pub initial_backoff: Duration,
6035    /// Maximum delay between retries.
6036    pub max_backoff: Duration,
6037}
6038
6039impl Default for WorkerRetryPolicy {
6040    fn default() -> Self {
6041        Self {
6042            max_retries: 5,
6043            initial_backoff: Duration::from_millis(100),
6044            max_backoff: Duration::from_secs(5),
6045        }
6046    }
6047}
6048
6049#[derive(Clone, Debug)]
6050struct WorkerStorageAdmission {
6051    policy: WorkerRetryPolicy,
6052    stop: Arc<AtomicBool>,
6053}
6054
6055struct StopWorkerOnDrop(Arc<AtomicBool>);
6056
6057impl Drop for StopWorkerOnDrop {
6058    fn drop(&mut self) {
6059        self.0.store(true, Ordering::SeqCst);
6060    }
6061}
6062
6063async fn wait_for_worker_stop(stop: &AtomicBool) {
6064    while !stop.load(Ordering::SeqCst) {
6065        tokio::time::sleep(Duration::from_millis(100)).await;
6066    }
6067}
6068
6069#[derive(Clone, Copy, Debug, PartialEq, Eq)]
6070enum ManagedPollOutcome {
6071    Idle,
6072    Handled,
6073    Stop,
6074}
6075
6076#[derive(Clone)]
6077pub struct Worker {
6078    client: Client,
6079    worker_id: String,
6080    task_queue: String,
6081    workflows: HashMap<String, RegisteredWorkflow>,
6082    activities: HashMap<String, ActivityHandler>,
6083    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
6084    updates: HashMap<String, HashMap<String, UpdateHandler>>,
6085    max_concurrent_workflow_tasks: usize,
6086    max_concurrent_activity_tasks: usize,
6087    poll_timeout: Duration,
6088    heartbeat_interval: Duration,
6089    retry_policy: WorkerRetryPolicy,
6090    heartbeat_observer: Option<WorkerHeartbeatObserver>,
6091}
6092
6093impl Worker {
6094    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
6095        Self {
6096            client,
6097            worker_id: default_worker_id(),
6098            task_queue: task_queue.into(),
6099            workflows: HashMap::new(),
6100            activities: HashMap::new(),
6101            queries: HashMap::new(),
6102            updates: HashMap::new(),
6103            max_concurrent_workflow_tasks: 10,
6104            max_concurrent_activity_tasks: 10,
6105            poll_timeout: Duration::from_secs(30),
6106            heartbeat_interval: Duration::from_secs(60),
6107            retry_policy: WorkerRetryPolicy::default(),
6108            heartbeat_observer: None,
6109        }
6110    }
6111
6112    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
6113        self.worker_id = worker_id.into();
6114        self
6115    }
6116
6117    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
6118        self.poll_timeout = timeout;
6119        self
6120    }
6121
6122    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
6123        self.heartbeat_interval = interval;
6124        self
6125    }
6126
6127    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
6128    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
6129        self.retry_policy = policy;
6130        self
6131    }
6132
6133    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
6134    where
6135        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
6136    {
6137        self.heartbeat_observer = Some(Arc::new(observer));
6138        self
6139    }
6140
6141    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
6142        self.max_concurrent_workflow_tasks = count.max(1);
6143        self
6144    }
6145
6146    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
6147        self.max_concurrent_activity_tasks = count.max(1);
6148        self
6149    }
6150
6151    /// Register a workflow handler.
6152    ///
6153    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
6154    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
6155    /// while acquiring or decoding a worker task remain worker-operation
6156    /// failures and do not get converted into workflow outcomes.
6157    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
6158    where
6159        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
6160        Fut: Future<Output = Result<Value>> + Send + 'static,
6161    {
6162        let handler = Arc::new(handler);
6163        self.workflows.insert(
6164            workflow_type.into(),
6165            RegisteredWorkflow {
6166                execute: Arc::new(move |ctx, input| {
6167                    let handler = Arc::clone(&handler);
6168                    Box::pin(async move {
6169                        let result = handler(ctx, input.into_json()?).await?;
6170                        AvroValue::from_serialize(&result)
6171                    })
6172                }),
6173                replay: None,
6174                state_type: None,
6175                definition_fingerprint: None,
6176            },
6177        );
6178    }
6179
6180    /// Register a workflow with one Serde request value and a Serde result.
6181    ///
6182    /// This is an ergonomic adapter over the same fixed Avro Value protocol as
6183    /// [`Worker::register_workflow_avro_value`]. It does not create or publish a
6184    /// workflow-specific schema. A task must contain zero arguments for a unit
6185    /// request or exactly one argument for every other request type.
6186    ///
6187    /// See the runnable
6188    /// [`hello_world` example](https://github.com/durable-workflow/sdk-rust/blob/main/examples/hello_world.rs)
6189    /// for typed workflow and activity contracts with retry and timeout policy.
6190    pub fn register_typed_workflow<I, O, F, Fut>(
6191        &mut self,
6192        workflow_type: impl Into<String>,
6193        handler: F,
6194    ) where
6195        I: DeserializeOwned + Send + 'static,
6196        O: Serialize + Send + 'static,
6197        F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6198        Fut: Future<Output = Result<O>> + Send + 'static,
6199    {
6200        let workflow_type = workflow_type.into();
6201        let handler_name = workflow_type.clone();
6202        let handler = Arc::new(handler);
6203        self.workflows.insert(
6204            workflow_type,
6205            RegisteredWorkflow {
6206                execute: Arc::new(move |ctx, input| {
6207                    let handler = Arc::clone(&handler);
6208                    let handler_name = handler_name.clone();
6209                    Box::pin(async move {
6210                        let input =
6211                            decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6212                        let result = handler(ctx, input).await?;
6213                        encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6214                    })
6215                }),
6216                replay: None,
6217                state_type: None,
6218                definition_fingerprint: None,
6219            },
6220        );
6221    }
6222
6223    /// Register a workflow on the lossless fixed Avro Value surface.
6224    pub fn register_workflow_avro_value<F, Fut>(
6225        &mut self,
6226        workflow_type: impl Into<String>,
6227        handler: F,
6228    ) where
6229        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6230        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6231    {
6232        self.workflows.insert(
6233            workflow_type.into(),
6234            RegisteredWorkflow {
6235                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6236                replay: None,
6237                state_type: None,
6238                definition_fingerprint: None,
6239            },
6240        );
6241    }
6242
6243    /// Register a workflow whose typed instance state can be reconstructed for queries.
6244    ///
6245    /// `state_factory` creates a fresh instance for every normal workflow task and
6246    /// query replay. The workflow handler is the single source of truth for state
6247    /// transitions: it updates [`WorkflowInstance`] after activities and signals
6248    /// resolve. Query replay runs this same handler over committed history and
6249    /// discards any commands it would emit.
6250    pub fn register_replayed_workflow<S, Factory, F, Fut>(
6251        &mut self,
6252        workflow_type: impl Into<String>,
6253        state_factory: Factory,
6254        handler: F,
6255    ) where
6256        S: Clone + Send + Sync + 'static,
6257        Factory: Fn() -> S + Send + Sync + 'static,
6258        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6259        Fut: Future<Output = Result<Value>> + Send + 'static,
6260    {
6261        let state_factory = Arc::new(state_factory);
6262        let handler = Arc::new(handler);
6263
6264        let execute_factory = Arc::clone(&state_factory);
6265        let execute_handler = Arc::clone(&handler);
6266        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6267            let state = WorkflowInstance::new(execute_factory());
6268            let handler = Arc::clone(&execute_handler);
6269            Box::pin(async move {
6270                let result = handler(ctx, input.into_json()?, state).await?;
6271                AvroValue::from_serialize(&result)
6272            }) as WorkflowFuture
6273        });
6274
6275        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6276            let state = WorkflowInstance::new(state_factory());
6277            let snapshot_state = state.clone();
6278            let snapshot: WorkflowStateSnapshot =
6279                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6280            let replay_handler = Arc::clone(&handler);
6281            let future = async move {
6282                let result = replay_handler(ctx, input.into_json()?, state).await?;
6283                AvroValue::from_serialize(&result)
6284            };
6285            ReplayedWorkflowInvocation {
6286                future: Box::pin(future),
6287                snapshot,
6288            }
6289        });
6290
6291        self.workflows.insert(
6292            workflow_type.into(),
6293            RegisteredWorkflow {
6294                execute,
6295                replay: Some(replay),
6296                state_type: Some(TypeId::of::<S>()),
6297                definition_fingerprint: None,
6298            },
6299        );
6300    }
6301
6302    /// Register a replayable workflow with one Serde request value and result.
6303    ///
6304    /// Normal task execution and instance-state query replay both decode and
6305    /// encode through the fixed Avro Value codec. The state factory and handler
6306    /// otherwise follow [`Worker::register_replayed_workflow`].
6307    pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6308        &mut self,
6309        workflow_type: impl Into<String>,
6310        state_factory: Factory,
6311        handler: F,
6312    ) where
6313        I: DeserializeOwned + Send + 'static,
6314        O: Serialize + Send + 'static,
6315        S: Clone + Send + Sync + 'static,
6316        Factory: Fn() -> S + Send + Sync + 'static,
6317        F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6318        Fut: Future<Output = Result<O>> + Send + 'static,
6319    {
6320        let workflow_type = workflow_type.into();
6321        let state_factory = Arc::new(state_factory);
6322        let handler = Arc::new(handler);
6323
6324        let execute_name = workflow_type.clone();
6325        let execute_factory = Arc::clone(&state_factory);
6326        let execute_handler = Arc::clone(&handler);
6327        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6328            let state = WorkflowInstance::new(execute_factory());
6329            let handler = Arc::clone(&execute_handler);
6330            let handler_name = execute_name.clone();
6331            Box::pin(async move {
6332                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6333                let result = handler(ctx, input, state).await?;
6334                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6335            }) as WorkflowFuture
6336        });
6337
6338        let replay_name = workflow_type.clone();
6339        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6340            let state = WorkflowInstance::new(state_factory());
6341            let snapshot_state = state.clone();
6342            let snapshot: WorkflowStateSnapshot =
6343                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6344            let handler = Arc::clone(&handler);
6345            let handler_name = replay_name.clone();
6346            let future = async move {
6347                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6348                let result = handler(ctx, input, state).await?;
6349                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6350            };
6351            ReplayedWorkflowInvocation {
6352                future: Box::pin(future),
6353                snapshot,
6354            }
6355        });
6356
6357        self.workflows.insert(
6358            workflow_type,
6359            RegisteredWorkflow {
6360                execute,
6361                replay: Some(replay),
6362                state_type: Some(TypeId::of::<S>()),
6363                definition_fingerprint: None,
6364            },
6365        );
6366    }
6367
6368    /// Bind a registered workflow to compile-time embedded source for safe redrive.
6369    ///
6370    /// Supply `include_str!` values for the workflow body and every helper whose
6371    /// behavior can affect replay. A handler re-registration clears the local
6372    /// identity; call this after registering the final handler. The server
6373    /// rejects reusing a worker ID when its prior fingerprint disappears.
6374    /// Workflows without source identity remain runnable but cannot be safely redriven.
6375    /// For example, pass `&[include_str!("workflows.rs")]` when the handler and
6376    /// replay-sensitive helpers are in a sibling `workflows.rs` file.
6377    pub fn set_workflow_definition_sources(
6378        &mut self,
6379        workflow_type: &str,
6380        sources: &[&str],
6381    ) -> Result<()> {
6382        if sources.is_empty() || sources.iter().any(|source| source.is_empty()) {
6383            return Err(Error::Codec(
6384                "workflow definition sources must be non-empty".to_string(),
6385            ));
6386        }
6387        let workflow = self.workflows.get_mut(workflow_type).ok_or_else(|| {
6388            Error::Codec(format!("workflow type {workflow_type:?} is not registered"))
6389        })?;
6390        let mut hasher = Sha256::new();
6391        hasher.update(b"durable-workflow-rust.workflow-definition.v1\0");
6392        hasher.update((workflow_type.len() as u64).to_be_bytes());
6393        hasher.update(workflow_type.as_bytes());
6394        for source in sources {
6395            hasher.update((source.len() as u64).to_be_bytes());
6396            hasher.update(source.as_bytes());
6397        }
6398        workflow.definition_fingerprint = Some(format!("sha256:{:x}", hasher.finalize()));
6399        Ok(())
6400    }
6401
6402    /// Register a replayable workflow on the lossless fixed Avro Value surface.
6403    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6404        &mut self,
6405        workflow_type: impl Into<String>,
6406        state_factory: Factory,
6407        handler: F,
6408    ) where
6409        S: Clone + Send + Sync + 'static,
6410        Factory: Fn() -> S + Send + Sync + 'static,
6411        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6412        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6413    {
6414        let state_factory = Arc::new(state_factory);
6415        let handler = Arc::new(handler);
6416
6417        let execute_factory = Arc::clone(&state_factory);
6418        let execute_handler = Arc::clone(&handler);
6419        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6420            let state = WorkflowInstance::new(execute_factory());
6421            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6422        });
6423
6424        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6425            let state = WorkflowInstance::new(state_factory());
6426            let snapshot_state = state.clone();
6427            let snapshot: WorkflowStateSnapshot =
6428                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6429            ReplayedWorkflowInvocation {
6430                future: Box::pin(handler(ctx, input, state)),
6431                snapshot,
6432            }
6433        });
6434
6435        self.workflows.insert(
6436            workflow_type.into(),
6437            RegisteredWorkflow {
6438                execute,
6439                replay: Some(replay),
6440                state_type: Some(TypeId::of::<S>()),
6441                definition_fingerprint: None,
6442            },
6443        );
6444    }
6445
6446    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6447    where
6448        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6449        Fut: Future<Output = Result<Value>> + Send + 'static,
6450    {
6451        let handler = Arc::new(handler);
6452        self.activities.insert(
6453            activity_type.into(),
6454            Arc::new(move |ctx, args| {
6455                let handler = Arc::clone(&handler);
6456                Box::pin(async move {
6457                    let result = handler(ctx, args.into_json()?).await?;
6458                    AvroValue::from_serialize(&result)
6459                })
6460            }),
6461        );
6462    }
6463
6464    /// Register an activity with one Serde request value and a Serde result.
6465    ///
6466    /// Inputs and results use the platform's fixed Avro Value schema. Shape
6467    /// mismatches and unsupported Serde values return [`Error::HandlerType`]
6468    /// with the activity name and Rust type.
6469    pub fn register_typed_activity<I, O, F, Fut>(
6470        &mut self,
6471        activity_type: impl Into<String>,
6472        handler: F,
6473    ) where
6474        I: DeserializeOwned + Send + 'static,
6475        O: Serialize + Send + 'static,
6476        F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6477        Fut: Future<Output = Result<O>> + Send + 'static,
6478    {
6479        let activity_type = activity_type.into();
6480        let handler_name = activity_type.clone();
6481        let handler = Arc::new(handler);
6482        self.activities.insert(
6483            activity_type,
6484            Arc::new(move |ctx, input| {
6485                let handler = Arc::clone(&handler);
6486                let handler_name = handler_name.clone();
6487                Box::pin(async move {
6488                    let input =
6489                        decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6490                    let result = handler(ctx, input).await?;
6491                    encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6492                })
6493            }),
6494        );
6495    }
6496
6497    /// Register an activity on the lossless fixed Avro Value surface.
6498    pub fn register_activity_avro_value<F, Fut>(
6499        &mut self,
6500        activity_type: impl Into<String>,
6501        handler: F,
6502    ) where
6503        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6504        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6505    {
6506        self.activities.insert(
6507            activity_type.into(),
6508            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6509        );
6510    }
6511
6512    /// Register a named, read-only query handler for a workflow type.
6513    ///
6514    /// The workflow type must also be registered with [`Worker::register_workflow`]
6515    /// before the worker runs. The handler receives only an immutable committed
6516    /// state snapshot and normalized query arguments.
6517    pub fn register_query<F, Fut>(
6518        &mut self,
6519        workflow_type: impl Into<String>,
6520        query_name: impl Into<String>,
6521        handler: F,
6522    ) where
6523        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6524        Fut: Future<Output = Result<Value>> + Send + 'static,
6525    {
6526        let handler = Arc::new(handler);
6527        self.queries
6528            .entry(workflow_type.into())
6529            .or_default()
6530            .insert(
6531                query_name.into(),
6532                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6533                    let handler = Arc::clone(&handler);
6534                    Box::pin(async move {
6535                        let result = handler(ctx, args.into_json()?).await?;
6536                        AvroValue::from_serialize(&result)
6537                    })
6538                })),
6539            );
6540    }
6541
6542    /// Register a query handler on the lossless fixed Avro Value surface.
6543    pub fn register_query_avro_value<F, Fut>(
6544        &mut self,
6545        workflow_type: impl Into<String>,
6546        query_name: impl Into<String>,
6547        handler: F,
6548    ) where
6549        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6550        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6551    {
6552        self.queries
6553            .entry(workflow_type.into())
6554            .or_default()
6555            .insert(
6556                query_name.into(),
6557                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6558            );
6559    }
6560
6561    /// Register a named query against deterministically replayed instance state.
6562    ///
6563    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
6564    /// same state type `S`. The handler receives an immutable, detached state
6565    /// clone, so successful and failed queries cannot affect workflow execution
6566    /// or the state reconstructed by a later query.
6567    pub fn register_replayed_query<S, F, Fut>(
6568        &mut self,
6569        workflow_type: impl Into<String>,
6570        query_name: impl Into<String>,
6571        handler: F,
6572    ) where
6573        S: Clone + Send + Sync + 'static,
6574        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6575        Fut: Future<Output = Result<Value>> + Send + 'static,
6576    {
6577        let handler = Arc::new(handler);
6578        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6579            let state = state.downcast::<S>().map_err(|_| {
6580                "registered query state type does not match the replayed workflow state".to_string()
6581            })?;
6582            let handler = Arc::clone(&handler);
6583            Ok(Box::pin(async move {
6584                let result = handler(ctx, state, args.into_json()?).await?;
6585                AvroValue::from_serialize(&result)
6586            }))
6587        });
6588
6589        self.queries
6590            .entry(workflow_type.into())
6591            .or_default()
6592            .insert(
6593                query_name.into(),
6594                RegisteredQuery::Replayed {
6595                    state_type: TypeId::of::<S>(),
6596                    handler: erased_handler,
6597                },
6598            );
6599    }
6600
6601    /// Register a replayed-state query on the lossless fixed Avro Value surface.
6602    pub fn register_replayed_query_avro_value<S, F, Fut>(
6603        &mut self,
6604        workflow_type: impl Into<String>,
6605        query_name: impl Into<String>,
6606        handler: F,
6607    ) where
6608        S: Clone + Send + Sync + 'static,
6609        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6610        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6611    {
6612        let handler = Arc::new(handler);
6613        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6614            let state = state.downcast::<S>().map_err(|_| {
6615                "registered query state type does not match the replayed workflow state".to_string()
6616            })?;
6617            Ok(Box::pin(handler(ctx, state, args)))
6618        });
6619
6620        self.queries
6621            .entry(workflow_type.into())
6622            .or_default()
6623            .insert(
6624                query_name.into(),
6625                RegisteredQuery::Replayed {
6626                    state_type: TypeId::of::<S>(),
6627                    handler: erased_handler,
6628                },
6629            );
6630    }
6631
6632    /// Register a synchronous workflow update handler.
6633    pub fn register_update<F, Fut>(
6634        &mut self,
6635        workflow_type: impl Into<String>,
6636        update_name: impl Into<String>,
6637        handler: F,
6638    ) where
6639        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6640        Fut: Future<Output = Result<Value>> + Send + 'static,
6641    {
6642        let handler = Arc::new(handler);
6643        self.updates
6644            .entry(workflow_type.into())
6645            .or_default()
6646            .insert(
6647                update_name.into(),
6648                Arc::new(move |ctx, args| {
6649                    let handler = Arc::clone(&handler);
6650                    Box::pin(async move {
6651                        let result = handler(ctx, args.into_json()?).await?;
6652                        AvroValue::from_serialize(&result)
6653                    })
6654                }),
6655            );
6656    }
6657
6658    /// Register an update handler on the lossless fixed Avro Value surface.
6659    pub fn register_update_avro_value<F, Fut>(
6660        &mut self,
6661        workflow_type: impl Into<String>,
6662        update_name: impl Into<String>,
6663        handler: F,
6664    ) where
6665        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6666        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6667    {
6668        self.updates
6669            .entry(workflow_type.into())
6670            .or_default()
6671            .insert(
6672                update_name.into(),
6673                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6674            );
6675    }
6676
6677    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6678        let mut command_contracts = serde_json::Map::new();
6679        for workflow_type in self.workflows.keys() {
6680            let mut queries = self
6681                .queries
6682                .get(workflow_type)
6683                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6684                .unwrap_or_default();
6685            queries.sort();
6686            let mut updates = self
6687                .updates
6688                .get(workflow_type)
6689                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6690                .unwrap_or_default();
6691            updates.sort();
6692            command_contracts.insert(
6693                workflow_type.clone(),
6694                json!({
6695                    "queries": queries,
6696                    "query_contracts": [],
6697                    "signals": [],
6698                    "signal_contracts": [],
6699                    "updates": updates,
6700                    "update_contracts": [],
6701                    "update_validators": [],
6702                }),
6703            );
6704        }
6705
6706        self.client
6707            .register_worker_with_definition_fingerprints(
6708                &self.worker_id,
6709                &self.task_queue,
6710                self.workflows.keys().cloned().collect(),
6711                self.activities.keys().cloned().collect(),
6712                self.max_concurrent_workflow_tasks,
6713                self.max_concurrent_activity_tasks,
6714                [
6715                    Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6716                    Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6717                    Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6718                    Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6719                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6720                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6721                    worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6722                        .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6723                ]
6724                .into_iter()
6725                .flatten()
6726                .collect(),
6727                Value::Object(command_contracts),
6728                Some(
6729                    self.workflows
6730                        .iter()
6731                        .filter_map(|(workflow_type, workflow)| {
6732                            workflow
6733                                .definition_fingerprint
6734                                .as_ref()
6735                                .map(|fingerprint| (workflow_type.clone(), fingerprint.clone()))
6736                        })
6737                        .collect(),
6738                ),
6739            )
6740            .await
6741    }
6742
6743    /// Run until shutdown or a terminal worker error occurs.
6744    ///
6745    /// Empty long-poll expirations do not stop the worker. Retryable poll and
6746    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
6747    /// authentication, protocol, and other non-retryable failures are returned.
6748    pub async fn run(&self) -> Result<()> {
6749        self.run_until(std::future::pending::<()>()).await
6750    }
6751
6752    /// Run until `shutdown` resolves or a terminal worker error occurs.
6753    ///
6754    /// This has the same liveness and terminal-error contract as [`Worker::run`].
6755    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6756    where
6757        F: Future<Output = ()>,
6758    {
6759        let stop = Arc::new(AtomicBool::new(false));
6760        let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6761        let worker = self.with_storage_admission(Arc::clone(&stop));
6762        let run = worker.run_with_storage_admission(Arc::clone(&stop));
6763        tokio::pin!(run);
6764        tokio::pin!(shutdown);
6765        tokio::select! {
6766            result = &mut run => result,
6767            _ = &mut shutdown => {
6768                stop.store(true, Ordering::SeqCst);
6769                run.await
6770            }
6771        }
6772    }
6773
6774    fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6775        let mut worker = self.clone();
6776        worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6777            policy: self.retry_policy,
6778            stop,
6779        });
6780        worker
6781    }
6782
6783    async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6784        let registration = self.register().await?;
6785        if !registration.registered {
6786            return Err(Error::WorkerLoop(format!(
6787                "worker registration for {:?} was not accepted",
6788                self.worker_id
6789            )));
6790        }
6791        let registered_worker_id = registration.worker_id.clone();
6792        let primary = self.run_registered_until(stop, registration).await;
6793        let deregistration = self
6794            .client
6795            .deregister_worker_registration(&registered_worker_id)
6796            .await;
6797
6798        match (primary, deregistration) {
6799            (Ok(()), Ok(_)) => Ok(()),
6800            (Ok(()), Err(deregistration)) => Err(deregistration),
6801            (Err(primary), Ok(_)) => Err(primary),
6802            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6803                primary: Box::new(primary),
6804                deregistration: Box::new(deregistration),
6805            }),
6806        }
6807    }
6808
6809    async fn run_registered_until(
6810        &self,
6811        stop: Arc<AtomicBool>,
6812        registration: RegisterWorkerResponse,
6813    ) -> Result<()> {
6814        let heartbeat_interval = Duration::from_secs(
6815            registration
6816                .heartbeat_interval_seconds
6817                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6818        );
6819        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
6820        // from the completion of the preceding attempt, including its bounded
6821        // retries. A fixed-epoch interval can leave an already-due tick queued
6822        // while an acknowledgement is slow, producing a catch-up heartbeat as
6823        // soon as that request completes.
6824        let heartbeat = tokio::time::sleep(Duration::ZERO);
6825        tokio::pin!(heartbeat);
6826        // Poll responses may already have leased server-side work by the time
6827        // they become ready, so each poller owns its responses through
6828        // completion or failure instead of racing raw polls in this select.
6829        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6830            let worker = self.clone();
6831            let stop = Arc::clone(&stop);
6832            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6833        });
6834        let mut activity_poller = (!self.activities.is_empty()).then(|| {
6835            let worker = self.clone();
6836            let stop = Arc::clone(&stop);
6837            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6838        });
6839        let mut query_poller = (!self.queries.is_empty()).then(|| {
6840            let worker = self.clone();
6841            let stop = Arc::clone(&stop);
6842            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6843        });
6844
6845        loop {
6846            tokio::select! {
6847                _ = wait_for_worker_stop(&stop) => {
6848                    stop.store(true, Ordering::SeqCst);
6849                    break;
6850                }
6851                _ = &mut heartbeat => {
6852                    let result = self.retry_worker_operation(|| {
6853                        self.client.heartbeat_worker(
6854                            &self.worker_id,
6855                            self.max_concurrent_workflow_tasks,
6856                            self.max_concurrent_activity_tasks,
6857                        )
6858                    }).await;
6859                    heartbeat
6860                        .as_mut()
6861                        .reset(tokio::time::Instant::now() + heartbeat_interval);
6862                    match result {
6863                        Ok(acknowledgement) => {
6864                            if let Some(observer) = &self.heartbeat_observer {
6865                                observer(&WorkerHeartbeatObservation {
6866                                    worker_id: self.worker_id.clone(),
6867                                    task_queue: self.task_queue.clone(),
6868                                    acknowledged_at_unix_millis: SystemTime::now()
6869                                        .duration_since(UNIX_EPOCH)
6870                                        .unwrap_or_default()
6871                                        .as_millis()
6872                                        .min(u64::MAX as u128)
6873                                        as u64,
6874                                    acknowledgement,
6875                                });
6876                            }
6877                        }
6878                        Err(error) => {
6879                            stop.store(true, Ordering::SeqCst);
6880                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6881                            return Err(error);
6882                        }
6883                    }
6884                }
6885                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6886                    workflow_poller = None;
6887                    let stopped_by_server = stop.load(Ordering::SeqCst);
6888                    stop.store(true, Ordering::SeqCst);
6889                    let poller_result = optional_poller_result("workflow", result);
6890                    let join_result =
6891                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6892                    poller_result?;
6893                    join_result?;
6894                    if stopped_by_server {
6895                        return Ok(());
6896                    }
6897                    return Err(Error::WorkerLoop(
6898                        "workflow poller stopped unexpectedly".to_string(),
6899                    ));
6900                }
6901                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6902                    activity_poller = None;
6903                    let stopped_by_server = stop.load(Ordering::SeqCst);
6904                    stop.store(true, Ordering::SeqCst);
6905                    let poller_result = optional_poller_result("activity", result);
6906                    let join_result =
6907                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6908                    poller_result?;
6909                    join_result?;
6910                    if stopped_by_server {
6911                        return Ok(());
6912                    }
6913                    return Err(Error::WorkerLoop(
6914                        "activity poller stopped unexpectedly".to_string(),
6915                    ));
6916                }
6917                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6918                    query_poller = None;
6919                    let stopped_by_server = stop.load(Ordering::SeqCst);
6920                    stop.store(true, Ordering::SeqCst);
6921                    let poller_result = optional_poller_result("query", result);
6922                    let join_result =
6923                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6924                    poller_result?;
6925                    join_result?;
6926                    if stopped_by_server {
6927                        return Ok(());
6928                    }
6929                    return Err(Error::WorkerLoop(
6930                        "query poller stopped unexpectedly".to_string(),
6931                    ));
6932                }
6933            }
6934        }
6935
6936        join_pollers(
6937            workflow_poller.take(),
6938            activity_poller.take(),
6939            query_poller.take(),
6940        )
6941        .await
6942    }
6943
6944    /// Poll and settle at most one task from each enabled task family.
6945    ///
6946    /// A workflow may reach its server-enforced run deadline while this worker
6947    /// holds a task. When the completion endpoint authoritatively rejects that
6948    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
6949    /// terminal `run_status=failed`, the workflow tick is considered settled:
6950    /// the late command was not recorded and cannot replace the terminal run.
6951    /// Every other completion rejection remains an error. This worker-level
6952    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
6953    /// only bounds how long a client waits for a result.
6954    ///
6955    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
6956    /// the original [`Error::Http`] status and response body.
6957    pub async fn run_once(&self) -> Result<usize> {
6958        let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
6959        let mut handled = 0;
6960        match worker.poll_workflow_once().await? {
6961            ManagedPollOutcome::Handled => handled += 1,
6962            ManagedPollOutcome::Stop => return Ok(handled),
6963            ManagedPollOutcome::Idle => {}
6964        }
6965        match worker.poll_activity_once().await? {
6966            ManagedPollOutcome::Handled => handled += 1,
6967            ManagedPollOutcome::Stop => return Ok(handled),
6968            ManagedPollOutcome::Idle => {}
6969        }
6970        if !self.queries.is_empty() {
6971            match worker.poll_query_once().await? {
6972                ManagedPollOutcome::Handled => handled += 1,
6973                ManagedPollOutcome::Stop => return Ok(handled),
6974                ManagedPollOutcome::Idle => {}
6975            }
6976        }
6977        Ok(handled)
6978    }
6979
6980    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6981        let poll_request_id = unique_request_id("rust-workflow-poll");
6982        let response = self
6983            .retry_worker_operation(|| {
6984                self.client.poll_workflow_task_response_with_request_id(
6985                    &self.worker_id,
6986                    &self.task_queue,
6987                    self.poll_timeout,
6988                    &poll_request_id,
6989                    0,
6990                )
6991            })
6992            .await;
6993        let Some(response) = self.settle_worker_poll_response(response).await? else {
6994            return Ok(ManagedPollOutcome::Idle);
6995        };
6996        if response.outcome().should_stop() {
6997            return Ok(ManagedPollOutcome::Stop);
6998        }
6999        let memo_updates_supported =
7000            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
7001        let Some(task) = response.task else {
7002            return Ok(ManagedPollOutcome::Idle);
7003        };
7004
7005        let task_id = task.task_id.clone();
7006        let attempt = task.workflow_task_attempt;
7007        let run_id = task.run_id.clone();
7008        let lease_owner = task
7009            .lease_owner
7010            .clone()
7011            .unwrap_or_else(|| self.worker_id.clone());
7012
7013        match self.execute_workflow_task_decision(task) {
7014            Ok(decision)
7015                if commands_use_workflow_memo_updates(&decision.commands)
7016                    && !memo_updates_supported =>
7017            {
7018                self.client
7019                    .fail_workflow_task(
7020                        &task_id,
7021                        &lease_owner,
7022                        attempt,
7023                        Error::WorkflowMemoUpdatesUnavailable.to_string(),
7024                    )
7025                    .await?;
7026            }
7027            Ok(decision) if decision.commands.is_empty() => {
7028                // A replay can consume a recorded pending durable command
7029                // without producing a new command. The standalone protocol
7030                // acknowledges that state through the typed waiting outcome;
7031                // an empty completion is rejected by servers that require at
7032                // least one executable command.
7033                self.client
7034                    .fail_workflow_task_with_type(
7035                        &task_id,
7036                        &lease_owner,
7037                        attempt,
7038                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
7039                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
7040                    )
7041                    .await?;
7042            }
7043            Ok(decision) => {
7044                let completion = self
7045                    .client
7046                    .complete_workflow_task_with_message_streams(
7047                        &task_id,
7048                        &lease_owner,
7049                        attempt,
7050                        decision.commands,
7051                        decision.message_stream_cursors,
7052                        decision.message_stream_waits,
7053                    )
7054                    .await;
7055                if let Err(error) = completion {
7056                    if !workflow_task_completion_is_terminal_timeout(
7057                        &error,
7058                        &task_id,
7059                        attempt,
7060                        run_id.as_deref(),
7061                    ) {
7062                        return Err(error);
7063                    }
7064                }
7065            }
7066            Err(error) => {
7067                self.client
7068                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
7069                    .await?;
7070            }
7071        }
7072
7073        Ok(ManagedPollOutcome::Handled)
7074    }
7075
7076    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7077        while !stop.load(Ordering::SeqCst) {
7078            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
7079                stop.store(true, Ordering::SeqCst);
7080                break;
7081            }
7082        }
7083
7084        Ok(())
7085    }
7086
7087    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
7088        let poll_request_id = unique_request_id("rust-activity-poll");
7089        let response = self
7090            .retry_worker_operation(|| {
7091                self.client.poll_activity_task_response_with_request_id(
7092                    &self.worker_id,
7093                    &self.task_queue,
7094                    self.poll_timeout,
7095                    &poll_request_id,
7096                    0,
7097                )
7098            })
7099            .await;
7100        let Some(response) = self.settle_worker_poll_response(response).await? else {
7101            return Ok(ManagedPollOutcome::Idle);
7102        };
7103        if response.outcome().should_stop() {
7104            return Ok(ManagedPollOutcome::Stop);
7105        }
7106        let Some(task) = response.task else {
7107            return Ok(ManagedPollOutcome::Idle);
7108        };
7109
7110        let task_id = task.task_id.clone();
7111        let attempt_id = task
7112            .activity_attempt_id
7113            .clone()
7114            .or(task.attempt_id.clone())
7115            .unwrap_or_default();
7116        let lease_owner = task
7117            .lease_owner
7118            .clone()
7119            .unwrap_or_else(|| self.worker_id.clone());
7120        let codec = task.payload_codec.clone();
7121        let result = self.execute_activity_task(task).await;
7122        match result {
7123            Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
7124            Ok(value) => {
7125                let completion = self
7126                    .client
7127                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
7128                    .await;
7129                if let Err(error) = completion {
7130                    if !activity_task_rejection_is_final(&error) {
7131                        return Err(error);
7132                    }
7133                }
7134            }
7135            Err(error) => {
7136                let failure = self
7137                    .client
7138                    .fail_activity_task(
7139                        &task_id,
7140                        &attempt_id,
7141                        &lease_owner,
7142                        error.to_string(),
7143                        false,
7144                    )
7145                    .await;
7146                if let Err(error) = failure {
7147                    if !activity_task_rejection_is_final(&error) {
7148                        return Err(error);
7149                    }
7150                }
7151            }
7152        }
7153
7154        Ok(ManagedPollOutcome::Handled)
7155    }
7156
7157    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7158        while !stop.load(Ordering::SeqCst) {
7159            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
7160                stop.store(true, Ordering::SeqCst);
7161                break;
7162            }
7163        }
7164
7165        Ok(())
7166    }
7167
7168    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
7169        let poll_request_id = unique_request_id("rust-query-poll");
7170        let response = self
7171            .retry_worker_operation(|| {
7172                self.client.poll_query_task_response_with_request_id(
7173                    &self.worker_id,
7174                    &self.task_queue,
7175                    self.poll_timeout,
7176                    &poll_request_id,
7177                    0,
7178                )
7179            })
7180            .await;
7181        let Some(response) = self.settle_worker_poll_response(response).await? else {
7182            return Ok(ManagedPollOutcome::Idle);
7183        };
7184        if response.outcome().should_stop() {
7185            return Ok(ManagedPollOutcome::Stop);
7186        }
7187        let Some(task) = response.task else {
7188            return Ok(ManagedPollOutcome::Idle);
7189        };
7190
7191        let query_task_id = task.query_task_id.clone();
7192        let attempt = task.query_task_attempt;
7193        let lease_owner = task
7194            .lease_owner
7195            .clone()
7196            .unwrap_or_else(|| self.worker_id.clone());
7197        let codec = task.payload_codec.clone();
7198
7199        match self.execute_query_task(task).await {
7200            Ok(value) => {
7201                let result_envelope = match encode_typed_envelope(&value, &codec) {
7202                    Ok(result_envelope) => result_envelope,
7203                    Err(error) => {
7204                        let failure = self
7205                            .client
7206                            .fail_query_task(
7207                                &query_task_id,
7208                                &lease_owner,
7209                                attempt,
7210                                error.to_string(),
7211                                "query_result_encode_failed",
7212                                "QueryResultEncodeFailed",
7213                            )
7214                            .await;
7215                        if let Err(error) = failure {
7216                            if !query_task_rejection_is_final(&error) {
7217                                return Err(error);
7218                            }
7219                        }
7220                        return Ok(ManagedPollOutcome::Handled);
7221                    }
7222                };
7223
7224                if let Err(error) = self
7225                    .client
7226                    .complete_query_task_with_envelope(
7227                        &query_task_id,
7228                        &lease_owner,
7229                        attempt,
7230                        value.clone().into_json()?,
7231                        result_envelope,
7232                    )
7233                    .await
7234                {
7235                    if !query_task_rejection_is_final(&error) {
7236                        return Err(error);
7237                    }
7238                }
7239            }
7240            Err(failure) => {
7241                let result = self
7242                    .client
7243                    .fail_query_task(
7244                        &query_task_id,
7245                        &lease_owner,
7246                        attempt,
7247                        failure.message,
7248                        failure.reason,
7249                        failure.failure_type,
7250                    )
7251                    .await;
7252                if let Err(error) = result {
7253                    if !query_task_rejection_is_final(&error) {
7254                        return Err(error);
7255                    }
7256                }
7257            }
7258        }
7259
7260        Ok(ManagedPollOutcome::Handled)
7261    }
7262
7263    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7264        while !stop.load(Ordering::SeqCst) {
7265            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
7266                stop.store(true, Ordering::SeqCst);
7267                break;
7268            }
7269        }
7270
7271        Ok(())
7272    }
7273
7274    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7275    where
7276        F: FnMut() -> Fut,
7277        Fut: Future<Output = Result<T>>,
7278    {
7279        let mut retries = 0;
7280
7281        loop {
7282            match operation().await {
7283                Err(error)
7284                    if worker_operation_is_retryable(&error)
7285                        && retries < self.retry_policy.max_retries =>
7286                {
7287                    retries += 1;
7288                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7289                }
7290                result => return result,
7291            }
7292        }
7293    }
7294
7295    async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7296        match response {
7297            Ok(response) => Ok(Some(response)),
7298            Err(error) => {
7299                let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7300                    return Err(error);
7301                };
7302                let minimum_delay = self
7303                    .retry_policy
7304                    .initial_backoff
7305                    .max(Duration::from_millis(1));
7306                let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7307                tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7308                Ok(None)
7309            }
7310        }
7311    }
7312
7313    async fn execute_query_task(
7314        &self,
7315        mut task: QueryTask,
7316    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7317        validate_query_task_payloads(&task).map_err(|error| {
7318            QueryTaskExecutionFailure::new(
7319                "query_payload_decode_failed",
7320                error.to_string(),
7321                "QueryPayloadDecodeFailed",
7322            )
7323        })?;
7324
7325        if !self.workflows.contains_key(&task.workflow_type) {
7326            return Err(QueryTaskExecutionFailure::new(
7327                "query_workflow_type_not_registered",
7328                format!("no workflow registered for type {:?}", task.workflow_type),
7329                "WorkflowTypeNotRegistered",
7330            ));
7331        }
7332
7333        let Some(handlers) = self.queries.get(&task.workflow_type) else {
7334            return Err(QueryTaskExecutionFailure::new(
7335                "query_handler_unavailable",
7336                format!(
7337                    "query handlers are unavailable for workflow type {:?}",
7338                    task.workflow_type
7339                ),
7340                "QueryHandlerUnavailable",
7341            ));
7342        };
7343        let Some(query) = handlers.get(&task.query_name) else {
7344            return Err(QueryTaskExecutionFailure::new(
7345                "rejected_unknown_query",
7346                format!("unknown query {:?}", task.query_name),
7347                "QueryFailed",
7348            ));
7349        };
7350
7351        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7352            .map_err(|error| {
7353            QueryTaskExecutionFailure::new(
7354                "query_payload_decode_failed",
7355                format!("cannot decode query arguments: {error}"),
7356                "QueryPayloadDecodeFailed",
7357            )
7358        })?;
7359        let workflow_input_typed =
7360            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7361                .map_err(|error| {
7362                    QueryTaskExecutionFailure::new(
7363                        "query_workflow_state_unavailable",
7364                        format!("cannot decode workflow start input: {error}"),
7365                        "QueryWorkflowStateUnavailable",
7366                    )
7367                })?;
7368        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7369            QueryTaskExecutionFailure::new(
7370                "query_workflow_state_unavailable",
7371                format!("cannot project workflow start input: {error}"),
7372                "QueryWorkflowStateUnavailable",
7373            )
7374        })?;
7375        hydrate_query_history_from_export(&mut task).map_err(|error| {
7376            QueryTaskExecutionFailure::new(
7377                "query_workflow_state_unavailable",
7378                format!("cannot restore query history snapshot: {error}"),
7379                "QueryWorkflowStateUnavailable",
7380            )
7381        })?;
7382        enrich_query_history_from_export(&mut task).map_err(|error| {
7383            QueryTaskExecutionFailure::new(
7384                "query_workflow_state_unavailable",
7385                format!("cannot restore compact query history payloads: {error}"),
7386                "QueryWorkflowStateUnavailable",
7387            )
7388        })?;
7389        let signal_events = query_signal_events(&task).map_err(|error| {
7390            QueryTaskExecutionFailure::new(
7391                "query_workflow_state_unavailable",
7392                format!("cannot decode committed workflow signals: {error}"),
7393                "QueryWorkflowStateUnavailable",
7394            )
7395        })?;
7396        let history_events = Arc::new(std::mem::take(&mut task.history_events));
7397        let context = QueryContext {
7398            workflow_id: task.workflow_id,
7399            run_id: task.run_id,
7400            workflow_type: task.workflow_type.clone(),
7401            run_status: task.run_status,
7402            workflow_input,
7403            workflow_input_avro_value: workflow_input_typed.clone(),
7404            history_events: Arc::clone(&history_events),
7405            signal_events: Arc::new(signal_events),
7406        };
7407
7408        let future = match query {
7409            RegisteredQuery::Snapshot(handler) => handler(context, args),
7410            RegisteredQuery::Replayed {
7411                state_type,
7412                handler,
7413            } => {
7414                let workflow = self
7415                    .workflows
7416                    .get(&task.workflow_type)
7417                    .expect("workflow registration was checked above");
7418                if workflow.state_type != Some(*state_type) {
7419                    return Err(QueryTaskExecutionFailure::new(
7420                        "query_workflow_state_unavailable",
7421                        "replayed query state type does not match its workflow registration",
7422                        "QueryWorkflowStateUnavailable",
7423                    ));
7424                }
7425                let replay = workflow.replay.as_ref().ok_or_else(|| {
7426                    QueryTaskExecutionFailure::new(
7427                        "query_workflow_state_unavailable",
7428                        format!(
7429                            "workflow type {:?} is not registered for instance-state replay",
7430                            task.workflow_type
7431                        ),
7432                        "QueryWorkflowStateUnavailable",
7433                    )
7434                })?;
7435                let workflow_state = Arc::new(Mutex::new(
7436                    WorkflowState::new_with_identity(
7437                        history_events.as_ref().clone(),
7438                        context.workflow_id.clone(),
7439                        context.run_id.clone(),
7440                        self.task_queue.clone(),
7441                        task.payload_codec,
7442                        None,
7443                    )
7444                    .map_err(|error| {
7445                        QueryTaskExecutionFailure::new(
7446                            "query_workflow_state_unavailable",
7447                            format!("workflow replay failed before query: {error}"),
7448                            "QueryWorkflowStateUnavailable",
7449                        )
7450                    })?,
7451                ));
7452                let workflow_context = WorkflowContext {
7453                    state: workflow_state,
7454                };
7455                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7456                let mut cx = TaskContext::from_waker(noop_waker_ref());
7457                match invocation.future.as_mut().poll(&mut cx) {
7458                    Poll::Ready(Ok(_)) => {
7459                        workflow_context
7460                            .ensure_history_consumed()
7461                            .map_err(|error| {
7462                                QueryTaskExecutionFailure::new(
7463                                    "query_workflow_state_unavailable",
7464                                    format!("workflow replay failed before query: {error}"),
7465                                    "QueryWorkflowStateUnavailable",
7466                                )
7467                            })?;
7468                    }
7469                    Poll::Ready(Err(error)) => {
7470                        return Err(QueryTaskExecutionFailure::new(
7471                            "query_workflow_state_unavailable",
7472                            format!("workflow replay failed before query: {error}"),
7473                            "QueryWorkflowStateUnavailable",
7474                        ));
7475                    }
7476                    Poll::Pending => {
7477                        let commands = workflow_context.take_commands().map_err(|error| {
7478                            QueryTaskExecutionFailure::new(
7479                                "query_workflow_state_unavailable",
7480                                format!("workflow replay failed before query: {error}"),
7481                                "QueryWorkflowStateUnavailable",
7482                            )
7483                        })?;
7484                        if commands.is_empty()
7485                            && !workflow_context
7486                                .matched_recorded_pending()
7487                                .map_err(|error| {
7488                                    QueryTaskExecutionFailure::new(
7489                                        "query_workflow_state_unavailable",
7490                                        format!("workflow replay failed before query: {error}"),
7491                                        "QueryWorkflowStateUnavailable",
7492                                    )
7493                                })?
7494                        {
7495                            return Err(QueryTaskExecutionFailure::new(
7496                                "query_workflow_state_unavailable",
7497                                "workflow replay yielded without a durable command",
7498                                "QueryWorkflowStateUnavailable",
7499                            ));
7500                        }
7501                    }
7502                }
7503                let state = (invocation.snapshot)().map_err(|error| {
7504                    QueryTaskExecutionFailure::new(
7505                        "query_workflow_state_unavailable",
7506                        format!("cannot snapshot replayed workflow state: {error}"),
7507                        "QueryWorkflowStateUnavailable",
7508                    )
7509                })?;
7510                handler(context, state, args).map_err(|message| {
7511                    QueryTaskExecutionFailure::new(
7512                        "query_workflow_state_unavailable",
7513                        message,
7514                        "QueryWorkflowStateUnavailable",
7515                    )
7516                })?
7517            }
7518        };
7519
7520        future.await.map_err(|error| {
7521            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7522        })
7523    }
7524
7525    #[cfg(test)]
7526    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7527        Ok(self.execute_workflow_task_decision(task)?.commands)
7528    }
7529
7530    fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7531        validate_workflow_task_payloads(&task)?;
7532
7533        if let Some(update_id) = task
7534            .workflow_update_id
7535            .as_deref()
7536            .filter(|update_id| !update_id.is_empty())
7537        {
7538            return self
7539                .execute_update_task(&task, update_id)
7540                .map(WorkflowTaskDecision::without_message_streams);
7541        }
7542
7543        let workflow = self
7544            .workflows
7545            .get(&task.workflow_type)
7546            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7547        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7548        let resume_signal = decode_resume_signal(&task)?;
7549        let history_budget = WorkflowHistoryBudget {
7550            event_count: task
7551                .total_history_events
7552                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7553            size_bytes: task.history_size_bytes,
7554            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7555            pressure: task.history_budget_pressure.clone(),
7556        };
7557        let workflow_command_identity = task
7558            .workflow_command_id
7559            .clone()
7560            .filter(|identity| !identity.is_empty())
7561            .unwrap_or_default();
7562        let mut workflow_state = WorkflowState::new_with_identity(
7563            task.history_events,
7564            task.workflow_id,
7565            task.run_id,
7566            self.task_queue.clone(),
7567            task.payload_codec.clone(),
7568            resume_signal,
7569        )?;
7570        workflow_state.history_budget = history_budget;
7571        workflow_state.workflow_command_identity = workflow_command_identity;
7572        workflow_state.cancel_requested = task.cancel_requested;
7573        let state = Arc::new(Mutex::new(workflow_state));
7574        let ctx = WorkflowContext { state };
7575        let mut future = (workflow.execute)(ctx.clone(), input);
7576        let mut cx = TaskContext::from_waker(noop_waker_ref());
7577
7578        match future.as_mut().poll(&mut cx) {
7579            Poll::Ready(Ok(result)) => {
7580                ctx.ensure_history_consumed()?;
7581                let result = encode_typed_envelope(&result, &task.payload_codec)?;
7582                let mut commands = ctx.take_commands()?;
7583                commands.push(json!({
7584                    "type": "complete_workflow",
7585                    "result": result
7586                }));
7587                self.message_stream_decision(&ctx, commands)
7588            }
7589            Poll::Ready(Err(error)) => {
7590                if let Error::ContinueAsNew(request) = error {
7591                    let mut commands = ctx.take_commands()?;
7592                    if let Some(command) = ctx.continue_as_new_command(request)? {
7593                        commands.push(command);
7594                    }
7595                    ctx.ensure_history_consumed()?;
7596                    return self.message_stream_decision(&ctx, commands);
7597                }
7598                if workflow_task_integrity_error(&error) {
7599                    // Replay and protocol failures describe the workflow-task
7600                    // decision itself. Preserve their specific failure reason
7601                    // instead of replacing it with the derivative fact that
7602                    // recorded commands remain unconsumed.
7603                    return Err(error);
7604                }
7605                // A handler error must not hide a committed durable command that
7606                // upgraded workflow code no longer consumes.
7607                ctx.ensure_history_consumed()?;
7608                let mut commands = ctx.take_commands()?;
7609                commands.push(workflow_failure_command(
7610                    &error,
7611                    recorded_activity_failure_boundary(&ctx, &error),
7612                ));
7613                self.message_stream_decision(&ctx, commands)
7614            }
7615            Poll::Pending => {
7616                let commands = ctx.take_commands()?;
7617                if commands.is_empty() && !ctx.matched_recorded_pending()? {
7618                    Err(Error::WorkflowYieldedWithoutCommand)
7619                } else {
7620                    self.message_stream_decision(&ctx, commands)
7621                }
7622            }
7623        }
7624    }
7625
7626    fn message_stream_decision(
7627        &self,
7628        ctx: &WorkflowContext,
7629        commands: Vec<Value>,
7630    ) -> Result<WorkflowTaskDecision> {
7631        let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7632        Ok(WorkflowTaskDecision {
7633            commands,
7634            message_stream_cursors,
7635            message_stream_waits,
7636        })
7637    }
7638
7639    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7640        if !self.workflows.contains_key(&task.workflow_type) {
7641            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7642        }
7643
7644        let accepted = task.history_events.iter().rev().find_map(|event| {
7645            (event.event_type == "UpdateAccepted"
7646                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7647            .then_some(&event.payload)
7648        });
7649        let update_name = accepted
7650            .and_then(|payload| payload.get("update_name"))
7651            .and_then(Value::as_str)
7652            .or(task.update_name.as_deref())
7653            .unwrap_or_default();
7654        let Some(handler) = self
7655            .updates
7656            .get(&task.workflow_type)
7657            .and_then(|handlers| handlers.get(update_name))
7658        else {
7659            return Ok(vec![json!({
7660                "type": "fail_update",
7661                "update_id": update_id,
7662                "message": format!(
7663                    "no update handler is registered for {}.{update_name}",
7664                    task.workflow_type
7665                ),
7666                "exception_type": "UnknownUpdate",
7667                "non_retryable": true,
7668            })]);
7669        };
7670        let arguments = accepted
7671            .and_then(|payload| payload.get("arguments"))
7672            .or(task.arguments.as_ref());
7673        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7674        let context = QueryContext {
7675            workflow_id: task.workflow_id.clone(),
7676            run_id: task.run_id.clone(),
7677            workflow_type: task.workflow_type.clone(),
7678            run_status: Some("running".to_string()),
7679            workflow_input: Value::Null,
7680            workflow_input_avro_value: AvroValue::Null,
7681            history_events: Arc::new(task.history_events.clone()),
7682            signal_events: Arc::new(Vec::new()),
7683        };
7684        let mut future = handler(context, arguments);
7685        let mut cx = TaskContext::from_waker(noop_waker_ref());
7686
7687        match future.as_mut().poll(&mut cx) {
7688            Poll::Ready(Ok(result)) => Ok(vec![json!({
7689                "type": "complete_update",
7690                "update_id": update_id,
7691                "result": encode_typed_envelope(&result, &task.payload_codec)?,
7692            })]),
7693            Poll::Ready(Err(error)) => Ok(vec![json!({
7694                "type": "fail_update",
7695                "update_id": update_id,
7696                "message": error.to_string(),
7697                "exception_type": "UpdateFailed",
7698                "non_retryable": true,
7699            })]),
7700            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7701        }
7702    }
7703
7704    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7705        validate_activity_task_payloads(&task)?;
7706
7707        let handler = self
7708            .activities
7709            .get(&task.activity_type)
7710            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7711        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7712        let attempt_id = task
7713            .activity_attempt_id
7714            .clone()
7715            .or(task.attempt_id.clone())
7716            .unwrap_or_default();
7717        let lease_owner = task
7718            .lease_owner
7719            .clone()
7720            .unwrap_or_else(|| self.worker_id.clone());
7721        let ctx = ActivityContext {
7722            client: self.client.clone(),
7723            task_id: task.task_id,
7724            activity_attempt_id: attempt_id,
7725            lease_owner,
7726            activity_type: task.activity_type,
7727            attempt_number: task.attempt_number,
7728            task_queue: self.task_queue.clone(),
7729            worker_id: self.worker_id.clone(),
7730        };
7731
7732        handler(ctx, args).await
7733    }
7734}
7735
7736fn poller_result(
7737    kind: &str,
7738    result: std::result::Result<Result<()>, tokio::task::JoinError>,
7739) -> Result<()> {
7740    match result {
7741        Ok(result) => result,
7742        Err(error) => Err(Error::WorkerLoop(format!(
7743            "{kind} poller join error: {error}"
7744        ))),
7745    }
7746}
7747
7748fn optional_poller_result(
7749    kind: &str,
7750    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7751) -> Result<()> {
7752    match result {
7753        Some(result) => poller_result(kind, result),
7754        None => Ok(()),
7755    }
7756}
7757
7758async fn join_pollers(
7759    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7760    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7761    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7762) -> Result<()> {
7763    let mut first_error = None;
7764
7765    if let Some(handle) = workflow_poller {
7766        if let Err(error) = poller_result("workflow", handle.await) {
7767            first_error.get_or_insert(error);
7768        }
7769    }
7770
7771    if let Some(handle) = activity_poller {
7772        if let Err(error) = poller_result("activity", handle.await) {
7773            first_error.get_or_insert(error);
7774        }
7775    }
7776
7777    if let Some(handle) = query_poller {
7778        if let Err(error) = poller_result("query", handle.await) {
7779            first_error.get_or_insert(error);
7780        }
7781    }
7782
7783    if let Some(error) = first_error {
7784        Err(error)
7785    } else {
7786        Ok(())
7787    }
7788}
7789
7790fn default_worker_id() -> String {
7791    let millis = SystemTime::now()
7792        .duration_since(UNIX_EPOCH)
7793        .unwrap_or_default()
7794        .as_millis();
7795    format!("rust-worker-{}-{millis}", std::process::id())
7796}
7797
7798fn percent_encode_path_segment(segment: &str) -> String {
7799    const HEX: &[u8; 16] = b"0123456789ABCDEF";
7800    let mut encoded = String::with_capacity(segment.len());
7801
7802    for byte in segment.bytes() {
7803        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7804            encoded.push(char::from(byte));
7805        } else {
7806            encoded.push('%');
7807            encoded.push(char::from(HEX[(byte >> 4) as usize]));
7808            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7809        }
7810    }
7811
7812    encoded
7813}
7814
7815fn unique_request_id(prefix: &str) -> String {
7816    let nanos = SystemTime::now()
7817        .duration_since(UNIX_EPOCH)
7818        .unwrap_or_default()
7819        .as_nanos();
7820    format!("{prefix}-{}-{nanos}", std::process::id())
7821}
7822
7823#[derive(Debug)]
7824struct QueryTaskExecutionFailure {
7825    reason: String,
7826    message: String,
7827    failure_type: String,
7828}
7829
7830impl QueryTaskExecutionFailure {
7831    fn new(
7832        reason: impl Into<String>,
7833        message: impl Into<String>,
7834        failure_type: impl Into<String>,
7835    ) -> Self {
7836        Self {
7837            reason: reason.into(),
7838            message: message.into(),
7839            failure_type: failure_type.into(),
7840        }
7841    }
7842}
7843
7844/// Typed local state owned by one deterministic workflow invocation.
7845///
7846/// Use [`WorkflowInstance::update`] for the same state transitions during
7847/// ordinary execution and replay. A replayed query receives a detached
7848/// immutable `Arc<S>` rather than this mutation-capable handle.
7849#[derive(Clone, Debug)]
7850pub struct WorkflowInstance<S> {
7851    state: Arc<Mutex<S>>,
7852}
7853
7854impl<S> WorkflowInstance<S> {
7855    fn new(state: S) -> Self {
7856        Self {
7857            state: Arc::new(Mutex::new(state)),
7858        }
7859    }
7860
7861    /// Read the current workflow-instance state without changing it.
7862    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7863        let state = self
7864            .state
7865            .lock()
7866            .map_err(|_| Error::WorkflowStatePoisoned)?;
7867        Ok(reader(&state))
7868    }
7869
7870    /// Apply one deterministic workflow-instance state transition.
7871    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7872        let mut state = self
7873            .state
7874            .lock()
7875            .map_err(|_| Error::WorkflowStatePoisoned)?;
7876        Ok(transition(&mut state))
7877    }
7878}
7879
7880impl<S: Clone> WorkflowInstance<S> {
7881    fn snapshot(&self) -> Result<S> {
7882        self.read(Clone::clone)
7883    }
7884}
7885
7886#[derive(Clone, Debug, PartialEq)]
7887pub struct MessageStreamMessage {
7888    pub stream_name: String,
7889    pub message_id: String,
7890    pub position: u64,
7891    pub arguments: Vec<AvroValue>,
7892}
7893
7894#[derive(Clone, Debug)]
7895pub struct MessageStream {
7896    ctx: WorkflowContext,
7897    name: String,
7898}
7899
7900impl MessageStream {
7901    /// Wait for one message, then return a bounded currently-available batch.
7902    pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7903        if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7904            return Err(Error::Codec(format!(
7905                "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7906            )));
7907        }
7908        loop {
7909            if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7910                return Ok(batch);
7911            }
7912
7913            self.ctx.record_message_stream_wait(&self.name)?;
7914            let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7915            let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7916            self.ctx.buffer_message_stream_delivery(arguments)?;
7917            if let Some(sequence) = replay_wait_sequence {
7918                self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7919            }
7920        }
7921    }
7922
7923    pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7924        self.receive(1)
7925            .await?
7926            .into_iter()
7927            .next()
7928            .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7929    }
7930}
7931
7932#[derive(Clone, Debug)]
7933pub struct WorkflowContext {
7934    state: Arc<Mutex<WorkflowState>>,
7935}
7936
7937fn valid_memo_key(key: &str) -> bool {
7938    let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7939
7940    !key.is_empty()
7941        && key.len() <= 64
7942        && (numeric_candidate.is_empty()
7943            || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7944        && key
7945            .bytes()
7946            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7947}
7948
7949fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7950    BASE64
7951        .decode(encode_avro_value(value)?.blob)
7952        .map(|bytes| bytes.len())
7953        .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7954}
7955
7956fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7957    let AvroValue::Map(entries) = value else {
7958        return Err(Error::InvalidMemoUpdate(
7959            "entries must serialize to an Avro string-keyed map".to_string(),
7960        ));
7961    };
7962    if require_entries && entries.is_empty() {
7963        return Err(Error::InvalidMemoUpdate(
7964            "at least one entry is required".to_string(),
7965        ));
7966    }
7967    if entries.len() > MAX_MEMO_ENTRIES {
7968        return Err(Error::InvalidMemoUpdate(format!(
7969            "at most {MAX_MEMO_ENTRIES} entries are allowed"
7970        )));
7971    }
7972
7973    for (key, value) in &entries {
7974        if !valid_memo_key(&key) {
7975            return Err(Error::InvalidMemoUpdate(
7976                "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7977            ));
7978        }
7979        if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7980            return Err(Error::InvalidMemoUpdate(format!(
7981                "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7982            )));
7983        }
7984    }
7985
7986    let value = AvroValue::Map(entries);
7987    if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7988        return Err(Error::InvalidMemoUpdate(format!(
7989            "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7990        )));
7991    }
7992    Ok(value)
7993}
7994
7995fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7996    let object = envelope.as_object().ok_or_else(|| {
7997        Error::InvalidMemoUpdate(
7998            "history field must use the public {codec, blob} payload envelope".to_string(),
7999        )
8000    })?;
8001    if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
8002        return Err(Error::InvalidMemoUpdate(
8003            "history field must use exactly the public {codec, blob} payload envelope".to_string(),
8004        ));
8005    }
8006
8007    canonical_memo_entries(
8008        decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
8009        require_entries,
8010    )
8011}
8012
8013impl WorkflowContext {
8014    pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
8015        let name = name.into();
8016        if name.is_empty()
8017            || name.len() > 128
8018            || !name.bytes().all(|byte| {
8019                byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
8020            })
8021        {
8022            return Err(Error::Codec(
8023                "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
8024                    .to_string(),
8025            ));
8026        }
8027        Ok(MessageStream {
8028            ctx: self.clone(),
8029            name,
8030        })
8031    }
8032
8033    fn record_message_stream_wait(&self, name: &str) -> Result<()> {
8034        let mut state = self
8035            .state
8036            .lock()
8037            .map_err(|_| Error::WorkflowStatePoisoned)?;
8038        let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
8039        state
8040            .message_stream_waits
8041            .insert(name.to_string(), position);
8042        Ok(())
8043    }
8044
8045    fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
8046        let mut state = self
8047            .state
8048            .lock()
8049            .map_err(|_| Error::WorkflowStatePoisoned)?;
8050        let cursor = state
8051            .message_stream_cursors
8052            .get(&message.stream_name)
8053            .copied()
8054            .unwrap_or(0);
8055        if message.position <= cursor {
8056            return Ok(());
8057        }
8058        let pending = state
8059            .message_stream_messages
8060            .entry(message.stream_name.clone())
8061            .or_default();
8062        if pending.iter().any(|candidate| {
8063            candidate.position == message.position || candidate.message_id == message.message_id
8064        }) {
8065            return Ok(());
8066        }
8067        pending.push(message);
8068        pending.sort_by_key(|candidate| candidate.position);
8069        Ok(())
8070    }
8071
8072    fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
8073        if let Some(delivery) = decode_message_stream_delivery(arguments)? {
8074            match delivery {
8075                MessageStreamDelivery::Message(message) => {
8076                    let stream_name = message.stream_name.clone();
8077                    self.buffer_message_stream(message)?;
8078                    return Ok(Some(stream_name));
8079                }
8080                MessageStreamDelivery::Cursor {
8081                    stream_name,
8082                    through_position,
8083                } => self.apply_message_stream_cursor(&stream_name, through_position)?,
8084            }
8085        }
8086        Ok(None)
8087    }
8088
8089    fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
8090        let state = self
8091            .state
8092            .lock()
8093            .map_err(|_| Error::WorkflowStatePoisoned)?;
8094        Ok(match state.recorded_commands.get(state.command_cursor) {
8095            Some(RecordedCommand::SignalWait {
8096                sequence,
8097                signal_name,
8098                ..
8099            }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
8100            _ => None,
8101        })
8102    }
8103
8104    fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
8105        let (history, payload_codec) = {
8106            let state = self
8107                .state
8108                .lock()
8109                .map_err(|_| Error::WorkflowStatePoisoned)?;
8110            (
8111                Arc::clone(&state.history_events),
8112                state.payload_codec.clone(),
8113            )
8114        };
8115
8116        let Some(opened_index) = history.iter().position(|event| {
8117            event.event_type == "SignalWaitOpened"
8118                && durable_event_sequence(event) == Some(wait_sequence)
8119                && event.payload.get("signal_name").and_then(Value::as_str)
8120                    == Some(MESSAGE_STREAM_SIGNAL)
8121        }) else {
8122            return Ok(());
8123        };
8124        let boundary_index = history
8125            .iter()
8126            .enumerate()
8127            .skip(opened_index + 1)
8128            .find_map(|(index, event)| {
8129                (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
8130                    && is_authored_command_open_event(event))
8131                .then_some(index)
8132            })
8133            .unwrap_or(history.len());
8134
8135        for event in history[opened_index + 1..boundary_index]
8136            .iter()
8137            .filter(|event| {
8138                event.event_type == "SignalReceived"
8139                    && event.payload.get("signal_name").and_then(Value::as_str)
8140                        == Some(MESSAGE_STREAM_SIGNAL)
8141            })
8142        {
8143            let arguments = decode_signal_event_arguments(event, &payload_codec)?
8144                .into_iter()
8145                .map(AvroValue::into_json)
8146                .collect::<Result<Vec<_>>>()?;
8147            self.buffer_message_stream_delivery(arguments)?;
8148        }
8149        Ok(())
8150    }
8151
8152    fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
8153        let mut state = self
8154            .state
8155            .lock()
8156            .map_err(|_| Error::WorkflowStatePoisoned)?;
8157        let cursor = state
8158            .message_stream_cursors
8159            .entry(name.to_string())
8160            .or_default();
8161        *cursor = (*cursor).max(through_position);
8162        if let Some(pending) = state.message_stream_messages.get_mut(name) {
8163            pending.retain(|message| message.position > through_position);
8164        }
8165        Ok(())
8166    }
8167
8168    fn take_message_stream_batch(
8169        &self,
8170        name: &str,
8171        max_items: usize,
8172    ) -> Result<Option<Vec<MessageStreamMessage>>> {
8173        let mut state = self
8174            .state
8175            .lock()
8176            .map_err(|_| Error::WorkflowStatePoisoned)?;
8177        let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
8178        let pending = state
8179            .message_stream_messages
8180            .entry(name.to_string())
8181            .or_default();
8182        let count = contiguous_message_stream_count(pending, cursor, max_items);
8183        if count == 0 {
8184            return Ok(None);
8185        }
8186        let batch = pending.drain(..count).collect::<Vec<_>>();
8187        let position = batch.last().map(|message| message.position).unwrap_or(0);
8188        state
8189            .message_stream_cursors
8190            .insert(name.to_string(), position);
8191        state.message_stream_waits.remove(name);
8192        Ok(Some(batch))
8193    }
8194
8195    fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
8196        let state = self
8197            .state
8198            .lock()
8199            .map_err(|_| Error::WorkflowStatePoisoned)?;
8200        let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
8201        cursors.sort_by_key(|(name, _)| *name);
8202        let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
8203        waits.sort_by_key(|(name, _)| *name);
8204        Ok((
8205            cursors
8206                .into_iter()
8207                .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
8208                .collect(),
8209            waits
8210                .into_iter()
8211                .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
8212                .collect(),
8213        ))
8214    }
8215    /// Identity of the parent workflow currently being replayed.
8216    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
8217        let state = self
8218            .state
8219            .lock()
8220            .map_err(|_| Error::WorkflowStatePoisoned)?;
8221        Ok(WorkflowIdentity {
8222            workflow_id: state.workflow_id.clone(),
8223            run_id: state.run_id.clone(),
8224        })
8225    }
8226
8227    /// Return the server-published history budget for this workflow task.
8228    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
8229        let state = self
8230            .state
8231            .lock()
8232            .map_err(|_| Error::WorkflowStatePoisoned)?;
8233        Ok(state.history_budget.clone())
8234    }
8235
8236    /// Continue this workflow instance as a fresh run with replacement arguments.
8237    ///
8238    /// Return this value directly from the workflow handler. The worker converts
8239    /// it to the terminal protocol command only after replay has consumed every
8240    /// recorded durable command.
8241    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
8242        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
8243    }
8244
8245    /// Continue as new with optional workflow-type and task-queue overrides.
8246    pub fn continue_as_new_with_options<T: Serialize>(
8247        &self,
8248        options: ContinueAsNewOptions,
8249        args: T,
8250    ) -> Result<Value> {
8251        options.validate()?;
8252        Err(Error::ContinueAsNew(ContinueAsNewRequest {
8253            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
8254            options,
8255        }))
8256    }
8257
8258    pub fn activity<T: Serialize>(
8259        &self,
8260        activity_type: impl Into<String>,
8261        args: T,
8262    ) -> ActivityCall {
8263        self.activity_with_options(activity_type, ActivityOptions::new(), args)
8264    }
8265
8266    pub fn activity_on_queue<T, Q>(
8267        &self,
8268        activity_type: impl Into<String>,
8269        task_queue: Option<Q>,
8270        args: T,
8271    ) -> ActivityCall
8272    where
8273        T: Serialize,
8274        Q: Into<String>,
8275    {
8276        let mut options = ActivityOptions::new();
8277        options.task_queue = task_queue.map(Into::into);
8278        self.activity_with_options(activity_type, options, args)
8279    }
8280
8281    /// Schedule one durable activity with retry, routing, and timeout options.
8282    ///
8283    /// Options are validated before the command is emitted. Once the command is
8284    /// recorded, replay consumes the same activity lifecycle at this command
8285    /// position and never emits a duplicate schedule.
8286    ///
8287    /// ```no_run
8288    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
8289    /// # use std::time::Duration;
8290    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
8291    /// let result = ctx
8292    ///     .activity_with_options(
8293    ///         "charge-card",
8294    ///         ActivityOptions::new()
8295    ///             .task_queue("payments")
8296    ///             .retry_policy(
8297    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
8298    ///                     Duration::from_secs(1),
8299    ///                     2,
8300    ///                     Some(Duration::from_secs(30)),
8301    ///                 ),
8302    ///             )
8303    ///             .start_to_close_timeout(Duration::from_secs(60))
8304    ///             .schedule_to_close_timeout(Duration::from_secs(180))
8305    ///             .heartbeat_timeout(Duration::from_secs(15)),
8306    ///         json!([{"order_id": "order-42"}]),
8307    ///     )
8308    ///     .await;
8309    /// match result {
8310    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
8311    ///         "reason": failure.reason,
8312    ///         "timeout_kind": failure.timeout_kind,
8313    ///     })),
8314    ///     other => other,
8315    /// }
8316    /// # }
8317    /// ```
8318    pub fn activity_with_options<T: Serialize>(
8319        &self,
8320        activity_type: impl Into<String>,
8321        options: ActivityOptions,
8322        args: T,
8323    ) -> ActivityCall {
8324        ActivityCall {
8325            ctx: self.clone(),
8326            activity_type: activity_type.into(),
8327            options,
8328            args: Some(AvroValue::from_serialize(&args)),
8329            scheduled: false,
8330            parallel_group_path: Vec::new(),
8331        }
8332    }
8333
8334    pub async fn activity_avro_value<T: Serialize>(
8335        &self,
8336        activity_type: impl Into<String>,
8337        args: T,
8338    ) -> Result<AvroValue> {
8339        let mut call = self.activity(activity_type, args);
8340        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8341    }
8342
8343    pub async fn activity_avro_value_with_options<T: Serialize>(
8344        &self,
8345        activity_type: impl Into<String>,
8346        options: ActivityOptions,
8347        args: T,
8348    ) -> Result<AvroValue> {
8349        let mut call = self.activity_with_options(activity_type, options, args);
8350        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8351    }
8352
8353    /// Schedule an activity with a Serde request and decode its Serde result.
8354    pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8355    where
8356        I: Serialize,
8357        O: DeserializeOwned,
8358    {
8359        self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8360            .await
8361    }
8362
8363    /// Schedule an activity with options and decode its result into `O`.
8364    ///
8365    /// Both directions use the fixed Avro Value codec. In particular, this
8366    /// method does not deserialize the JSON-safe inspection projection returned
8367    /// by the dynamic [`ActivityCall`] future.
8368    pub async fn activity_typed_with_options<I, O>(
8369        &self,
8370        activity_type: impl Into<String>,
8371        options: ActivityOptions,
8372        args: I,
8373    ) -> Result<O>
8374    where
8375        I: Serialize,
8376        O: DeserializeOwned,
8377    {
8378        let activity_type = activity_type.into();
8379        let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8380            handler_type_error::<I>(
8381                HandlerKind::Activity,
8382                &activity_type,
8383                HandlerValueKind::Input,
8384                error.to_string(),
8385            )
8386        });
8387        let mut call = ActivityCall {
8388            ctx: self.clone(),
8389            activity_type: activity_type.clone(),
8390            options,
8391            args: Some(encoded),
8392            scheduled: false,
8393            parallel_group_path: Vec::new(),
8394        };
8395        let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8396        decode_handler_result(result, HandlerKind::Activity, &activity_type)
8397    }
8398
8399    /// Schedule and join a deterministic activity/child/timer group.
8400    ///
8401    /// Nested groups retain their input shape. Every durable leaf is scheduled
8402    /// before this future yields, results are assembled by declaration order,
8403    /// and a failure returns [`Error::ParallelFailed`] with typed cause,
8404    /// declaration path, stable group metadata, and completed siblings.
8405    pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8406        ParallelCall::new(self.clone(), operations)
8407    }
8408
8409    /// Alias for [`WorkflowContext::parallel`].
8410    pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8411        self.parallel(operations)
8412    }
8413
8414    /// Lossless fixed-Avro variant of [`WorkflowContext::parallel`].
8415    pub async fn parallel_avro_value(
8416        &self,
8417        operations: Vec<ParallelOperation>,
8418    ) -> Result<Vec<ParallelAvroResult>> {
8419        let mut call = self.parallel(operations);
8420        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8421    }
8422
8423    /// Start every durable operation and resume from the one winner persisted
8424    /// by Server. Non-winning operations continue and remain addressable.
8425    pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8426        let operations = operations
8427            .into_iter()
8428            .enumerate()
8429            .map(|(index, operation)| (SelectionKey::Index(index), operation))
8430            .collect();
8431        SelectCall::new(self.clone(), operations)
8432    }
8433
8434    /// Named-key variant of [`WorkflowContext::select`].
8435    pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8436    where
8437        K: Into<SelectionKey>,
8438    {
8439        SelectCall::new(
8440            self.clone(),
8441            operations
8442                .into_iter()
8443                .map(|(key, operation)| (key.into(), operation))
8444                .collect(),
8445        )
8446    }
8447
8448    /// Create a workflow-local deterministic compensation registry.
8449    pub fn saga(&self) -> Saga {
8450        Saga::new(self.clone())
8451    }
8452
8453    /// Whether the current workflow task carries a cooperative cancel request.
8454    pub fn is_cancellation_requested(&self) -> Result<bool> {
8455        let state = self
8456            .state
8457            .lock()
8458            .map_err(|_| Error::WorkflowStatePoisoned)?;
8459        Ok(state.cancel_requested)
8460    }
8461
8462    /// Raise a typed cooperative cancellation at an author-controlled point.
8463    ///
8464    /// Passing this result to [`Saga::finish`] compensates already registered
8465    /// forward steps before the cancellation remains the initiating outcome.
8466    pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8467        if self.is_cancellation_requested()? {
8468            return Err(Error::WorkflowCancellationRequested(
8469                WorkflowCancellationRequested,
8470            ));
8471        }
8472        Ok(())
8473    }
8474
8475    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8476        SignalCall {
8477            ctx: self.clone(),
8478            signal_name: signal_name.into(),
8479            runtime_reserved_allowed: false,
8480            opened_wait: false,
8481            matched_pending: false,
8482            parallel_group_path: Vec::new(),
8483        }
8484    }
8485
8486    fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8487        SignalCall {
8488            ctx: self.clone(),
8489            signal_name: signal_name.into(),
8490            runtime_reserved_allowed: true,
8491            opened_wait: false,
8492            matched_pending: false,
8493            parallel_group_path: Vec::new(),
8494        }
8495    }
8496
8497    pub async fn wait_signal_avro_value(
8498        &self,
8499        signal_name: impl Into<String>,
8500    ) -> Result<Vec<AvroValue>> {
8501        let mut call = self.wait_signal(signal_name);
8502        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8503    }
8504
8505    /// Return every committed signal argument list with the given name.
8506    ///
8507    /// This history-backed view is deterministic and is intended for
8508    /// condition predicates that must be re-evaluated after a signal while the
8509    /// workflow is blocked on [`WorkflowContext::wait_condition`].
8510    pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8511        self.signals_avro_value(signal_name)?
8512            .into_iter()
8513            .map(|arguments| {
8514                arguments
8515                    .into_iter()
8516                    .map(AvroValue::into_json)
8517                    .collect::<Result<Vec<_>>>()
8518            })
8519            .collect()
8520    }
8521
8522    /// Lossless fixed Avro Value view of committed signals with the given name.
8523    pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8524        let state = self
8525            .state
8526            .lock()
8527            .map_err(|_| Error::WorkflowStatePoisoned)?;
8528        state
8529            .history_events
8530            .iter()
8531            .filter(|event| {
8532                event.event_type == "SignalReceived"
8533                    && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8534            })
8535            .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8536            .collect()
8537    }
8538
8539    /// Return every committed update argument list with the given name.
8540    ///
8541    /// Accepted and applied records for the same update ID are de-duplicated.
8542    /// A Server task created after an update therefore replays the workflow and
8543    /// re-evaluates an open condition without application polling.
8544    pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8545        self.updates_avro_value(update_name)?
8546            .into_iter()
8547            .map(|arguments| {
8548                arguments
8549                    .into_iter()
8550                    .map(AvroValue::into_json)
8551                    .collect::<Result<Vec<_>>>()
8552            })
8553            .collect()
8554    }
8555
8556    /// Lossless fixed Avro Value view of committed updates with the given name.
8557    pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8558        let state = self
8559            .state
8560            .lock()
8561            .map_err(|_| Error::WorkflowStatePoisoned)?;
8562        let mut seen = Vec::new();
8563        let mut updates = Vec::new();
8564        for event in state.history_events.iter() {
8565            if !matches!(
8566                event.event_type.as_str(),
8567                "UpdateAccepted" | "UpdateApplied"
8568            ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8569                || event.payload.get("arguments").is_none()
8570            {
8571                continue;
8572            }
8573            if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8574                if seen.iter().any(|recorded| recorded == update_id) {
8575                    continue;
8576                }
8577                seen.push(update_id.to_string());
8578            }
8579            updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8580        }
8581        Ok(updates)
8582    }
8583
8584    /// Wait for a deterministic predicate to become true or for its durable
8585    /// timeout to elapse.
8586    ///
8587    /// Prefer [`wait_condition!`] for inline predicates so changes to the Rust
8588    /// predicate tokens automatically change the recorded definition
8589    /// fingerprint. Direct callers must provide an equally stable identity in
8590    /// [`ConditionWaitOptions`].
8591    pub fn wait_condition<F>(
8592        &self,
8593        options: ConditionWaitOptions,
8594        predicate: F,
8595    ) -> ConditionWaitCall
8596    where
8597        F: Fn() -> Result<bool> + Send + 'static,
8598    {
8599        ConditionWaitCall {
8600            ctx: self.clone(),
8601            options,
8602            predicate: Box::new(predicate),
8603            occurrence_id: None,
8604            opened_wait: false,
8605            parallel_group_path: Vec::new(),
8606        }
8607    }
8608
8609    /// Wait for server-backed durable time without blocking the worker executor.
8610    ///
8611    /// Polling this future emits one `start_timer` command and yields. The
8612    /// server records the deadline, so neither worker nor server restarts reset
8613    /// the wait. Replay resolves the future only from a `TimerScheduled` and
8614    /// `TimerFired` pair at the same position in the shared durable-command
8615    /// stream, with matching sequence, timer identity, and delay. Sub-second
8616    /// durations round up because protocol deadlines use whole seconds.
8617    ///
8618    /// ```no_run
8619    /// # use durable_workflow::{json, Client, Worker};
8620    /// # use std::time::Duration;
8621    /// # fn configure(client: Client) {
8622    /// let mut worker = Worker::new(client, "rust-workers");
8623    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
8624    ///     ctx.sleep(Duration::from_secs(5)).await?;
8625    ///     Ok(json!({"status": "timer fired"}))
8626    /// });
8627    /// # }
8628    /// ```
8629    pub fn sleep(&self, duration: Duration) -> TimerCall {
8630        let delay_seconds = duration
8631            .as_secs()
8632            .checked_add(u64::from(duration.subsec_nanos() > 0));
8633        TimerCall {
8634            ctx: self.clone(),
8635            delay_seconds,
8636            scheduled: false,
8637            matched_pending: false,
8638            parallel_group_path: Vec::new(),
8639        }
8640    }
8641
8642    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
8643    pub fn start_timer(&self, duration: Duration) -> TimerCall {
8644        self.sleep(duration)
8645    }
8646
8647    /// Evaluate a non-deterministic callback once and durably record its typed value.
8648    ///
8649    /// On replay the callback is not invoked: the value is decoded from the
8650    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
8651    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
8652    /// small values that must remain fixed for the lifetime of a workflow run.
8653    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8654    where
8655        T: Serialize + DeserializeOwned,
8656        F: FnOnce() -> T,
8657    {
8658        {
8659            let mut state = self
8660                .state
8661                .lock()
8662                .map_err(|_| Error::WorkflowStatePoisoned)?;
8663            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8664                return match recorded {
8665                    RecordedCommand::SideEffect { sequence, value } => {
8666                        state.command_cursor += 1;
8667                        value.deserialize().map_err(|error| {
8668                            Error::NonDeterministicReplay(ReplayFailure::new(
8669                                "side_effect_type_mismatch",
8670                                Some(sequence),
8671                                Some(std::any::type_name::<T>().to_string()),
8672                                Some(error.to_string()),
8673                                "recorded side-effect value is incompatible with the requested Rust type",
8674                            ))
8675                        })
8676                    }
8677                    other => Err(command_mismatch(&other, "side effect")),
8678                };
8679            }
8680        }
8681
8682        let value = callback();
8683        let avro_value = AvroValue::from_serialize(&value)?;
8684        let mut state = self
8685            .state
8686            .lock()
8687            .map_err(|_| Error::WorkflowStatePoisoned)?;
8688        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8689        state.commands.push(json!({
8690            "type": "record_side_effect",
8691            "result": result,
8692        }));
8693        Ok(value)
8694    }
8695
8696    /// Record or replay a lossless fixed Avro Value side effect.
8697    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8698    where
8699        F: FnOnce() -> AvroValue,
8700    {
8701        {
8702            let mut state = self
8703                .state
8704                .lock()
8705                .map_err(|_| Error::WorkflowStatePoisoned)?;
8706            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8707                return match recorded {
8708                    RecordedCommand::SideEffect { value, .. } => {
8709                        state.command_cursor += 1;
8710                        Ok(value)
8711                    }
8712                    other => Err(command_mismatch(&other, "side effect")),
8713                };
8714            }
8715        }
8716
8717        let value = callback();
8718        let mut state = self
8719            .state
8720            .lock()
8721            .map_err(|_| Error::WorkflowStatePoisoned)?;
8722        let result = encode_typed_envelope(&value, &state.payload_codec)?;
8723        state.commands.push(json!({
8724            "type": "record_side_effect",
8725            "result": result,
8726        }));
8727        Ok(value)
8728    }
8729
8730    /// Append output items at a deterministic workflow command boundary.
8731    ///
8732    /// Stable idempotency keys are derived from the server-provided durable
8733    /// workflow command identity, command ordinal, and item index. Replay
8734    /// consumes the recorded side effect and never emits another append.
8735    pub fn append_workflow_stream(
8736        &self,
8737        stream_name: impl Into<String>,
8738        items: &[WorkflowStreamAppendItem],
8739        max_pending_items: Option<u64>,
8740    ) -> Result<()> {
8741        if items.is_empty() {
8742            return Err(Error::Codec(
8743                "workflow_stream_items_empty: append requires at least one item".to_string(),
8744            ));
8745        }
8746        if max_pending_items == Some(0) {
8747            return Err(Error::Codec(
8748                "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8749                    .to_string(),
8750            ));
8751        }
8752        let stream_name = stream_name.into();
8753        if stream_name.is_empty() {
8754            return Err(Error::Codec(
8755                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8756            ));
8757        }
8758
8759        let mut state = self
8760            .state
8761            .lock()
8762            .map_err(|_| Error::WorkflowStatePoisoned)?;
8763        let command_ordinal = state.workflow_stream_command_counter;
8764        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8765            state.workflow_stream_command_counter += 1;
8766            return match recorded {
8767                RecordedCommand::SideEffect { .. } => {
8768                    state.command_cursor += 1;
8769                    Ok(())
8770                }
8771                other => Err(command_mismatch(&other, "workflow stream append")),
8772            };
8773        }
8774
8775        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8776        state.workflow_stream_command_counter += 1;
8777        let wire_items = items
8778            .iter()
8779            .enumerate()
8780            .map(|(item_index, item)| {
8781                item.wire_value(Some(format!(
8782                    "dw-stream:{identity}:{command_ordinal}:{item_index}"
8783                )))
8784            })
8785            .collect::<Vec<_>>();
8786        let mut directive = json!({
8787            "operation": "append",
8788            "stream_name": stream_name,
8789            "command_identity": identity,
8790            "command_ordinal": command_ordinal,
8791            "items": wire_items,
8792        });
8793        if let Some(max_pending_items) = max_pending_items {
8794            directive["max_pending_items"] = json!(max_pending_items);
8795        }
8796        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8797        state.commands.push(json!({
8798            "type": "record_side_effect",
8799            "result": result,
8800            "workflow_stream": directive,
8801        }));
8802        Ok(())
8803    }
8804
8805    /// Close a run-scoped output stream at a deterministic command boundary.
8806    pub fn close_workflow_stream(
8807        &self,
8808        stream_name: impl Into<String>,
8809        retention_seconds: Option<u64>,
8810    ) -> Result<()> {
8811        self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8812    }
8813
8814    /// Mark a run-scoped output stream errored at a deterministic command boundary.
8815    pub fn error_workflow_stream(
8816        &self,
8817        stream_name: impl Into<String>,
8818        error_reason: impl Into<String>,
8819        retention_seconds: Option<u64>,
8820    ) -> Result<()> {
8821        let error_reason = error_reason.into();
8822        if error_reason.is_empty() {
8823            return Err(Error::Codec(
8824                "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8825            ));
8826        }
8827        self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8828    }
8829
8830    fn finish_workflow_stream(
8831        &self,
8832        stream_name: String,
8833        error_reason: Option<String>,
8834        retention_seconds: Option<u64>,
8835    ) -> Result<()> {
8836        if stream_name.is_empty() {
8837            return Err(Error::Codec(
8838                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8839            ));
8840        }
8841        if retention_seconds == Some(0) {
8842            return Err(Error::Codec(
8843                "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8844            ));
8845        }
8846        let mut state = self
8847            .state
8848            .lock()
8849            .map_err(|_| Error::WorkflowStatePoisoned)?;
8850        let command_ordinal = state.workflow_stream_command_counter;
8851        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8852            state.workflow_stream_command_counter += 1;
8853            return match recorded {
8854                RecordedCommand::SideEffect { .. } => {
8855                    state.command_cursor += 1;
8856                    Ok(())
8857                }
8858                other => Err(command_mismatch(&other, "workflow stream close")),
8859            };
8860        }
8861        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8862        state.workflow_stream_command_counter += 1;
8863        let mut directive = json!({
8864            "operation": if error_reason.is_some() { "error" } else { "close" },
8865            "stream_name": stream_name,
8866            "command_identity": identity,
8867            "command_ordinal": command_ordinal,
8868        });
8869        if let Some(error_reason) = error_reason {
8870            directive["error_reason"] = json!(error_reason);
8871        }
8872        if let Some(retention_seconds) = retention_seconds {
8873            directive["retention_seconds"] = json!(retention_seconds);
8874        }
8875        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8876        state.commands.push(json!({
8877            "type": "record_side_effect",
8878            "result": result,
8879            "workflow_stream": directive,
8880        }));
8881        Ok(())
8882    }
8883
8884    fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8885        let identity = state.workflow_command_identity.as_str();
8886        if identity.is_empty() {
8887            return Err(Error::MissingWorkflowCommandIdentity);
8888        }
8889        Ok(identity)
8890    }
8891
8892    /// Validate, emit, or replay a typed workflow search-attribute update.
8893    ///
8894    /// The command is non-blocking within a workflow decision, but its
8895    /// `SearchAttributesUpserted` event occupies the same deterministic command
8896    /// stream as activities, timers, conditions, and other durable operations.
8897    pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8898        update.validate()?;
8899        let (attributes, attribute_types) = update.into_wire_parts();
8900        let mut state = self
8901            .state
8902            .lock()
8903            .map_err(|_| Error::WorkflowStatePoisoned)?;
8904
8905        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8906            return match recorded {
8907                RecordedCommand::SearchAttributes {
8908                    sequence,
8909                    attributes: recorded_attributes,
8910                    attribute_types: recorded_attribute_types,
8911                } => {
8912                    if recorded_attributes != attributes {
8913                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8914                            "search_attribute_value_mismatch",
8915                            Some(sequence),
8916                            Some(recorded_attributes.to_string()),
8917                            Some(attributes.to_string()),
8918                            "search-attribute values differ from the recorded durable command",
8919                        )));
8920                    }
8921                    if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8922                        if recorded_types != attribute_types {
8923                            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8924                                "search_attribute_type_mismatch",
8925                                Some(sequence),
8926                                Some(json!(recorded_types).to_string()),
8927                                Some(json!(attribute_types).to_string()),
8928                                "search-attribute declared types differ from the recorded durable command",
8929                            )));
8930                        }
8931                    }
8932                    state.command_cursor += 1;
8933                    Ok(())
8934                }
8935                other => Err(command_mismatch(&other, "search-attribute update")),
8936            };
8937        }
8938
8939        let mut command = serde_json::Map::from_iter([
8940            ("type".to_string(), json!("upsert_search_attributes")),
8941            ("attributes".to_string(), attributes),
8942        ]);
8943        if !attribute_types.is_empty() {
8944            command.insert("attribute_types".to_string(), json!(attribute_types));
8945        }
8946        state.commands.push(Value::Object(command));
8947        Ok(())
8948    }
8949
8950    /// Record a UUIDv4 once and return the same UUID on every replay.
8951    pub fn uuid_v4(&self) -> Result<Uuid> {
8952        self.side_effect(Uuid::new_v4)
8953    }
8954
8955    /// Select the newest supported version for a change, or replay the version
8956    /// already committed for that stable change ID.
8957    pub fn get_version(
8958        &self,
8959        change_id: impl Into<String>,
8960        min_supported: i32,
8961        max_supported: i32,
8962    ) -> Result<i32> {
8963        let change_id = change_id.into();
8964        if change_id.trim().is_empty() {
8965            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8966                "version_change_id_invalid",
8967                None,
8968                Some("non-empty change ID".to_string()),
8969                Some(change_id),
8970                "version markers require a stable non-empty change ID",
8971            )));
8972        }
8973        if min_supported > max_supported {
8974            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8975                "version_range_invalid",
8976                None,
8977                Some("min_supported <= max_supported".to_string()),
8978                Some(format!("{min_supported}..={max_supported}")),
8979                "version marker supported range is invalid",
8980            )));
8981        }
8982
8983        let mut state = self
8984            .state
8985            .lock()
8986            .map_err(|_| Error::WorkflowStatePoisoned)?;
8987        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8988            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8989            return Ok(version);
8990        }
8991
8992        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8993            return match recorded {
8994                RecordedCommand::VersionMarker {
8995                    sequence,
8996                    change_id: recorded_change_id,
8997                    version,
8998                    ..
8999                } => {
9000                    if recorded_change_id != change_id {
9001                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9002                            "version_change_id_mismatch",
9003                            Some(sequence),
9004                            Some(recorded_change_id),
9005                            Some(change_id),
9006                            "recorded version marker change ID differs from current workflow code",
9007                        )));
9008                    }
9009                    ensure_version_supported(
9010                        &change_id,
9011                        version,
9012                        min_supported,
9013                        max_supported,
9014                        sequence,
9015                    )?;
9016                    state.command_cursor += 1;
9017                    state.version_markers.insert(change_id, (version, sequence));
9018                    Ok(version)
9019                }
9020                other => Err(command_mismatch(
9021                    &other,
9022                    format!("version marker:{change_id}"),
9023                )),
9024            };
9025        }
9026
9027        let version = max_supported;
9028        state.commands.push(json!({
9029            "type": "record_version_marker",
9030            "change_id": change_id,
9031            "version": version,
9032            "min_supported": min_supported,
9033            "max_supported": max_supported,
9034        }));
9035        // Sequence numbers are assigned by the server. Zero identifies a marker
9036        // selected in this uncommitted decision batch for duplicate-call checks.
9037        state.version_markers.insert(change_id, (version, 0));
9038        Ok(version)
9039    }
9040
9041    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
9042    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
9043        Ok(self.get_version(change_id, -1, 1)? == 1)
9044    }
9045
9046    /// Keep a patch marker in history after the legacy branch has been removed.
9047    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
9048        self.get_version(change_id, -1, 1).map(|_| ())
9049    }
9050
9051    /// Merge non-indexed workflow memo metadata through durable history.
9052    ///
9053    /// Avro `null` deletes a key. The SDK encodes the complete patch in the
9054    /// public Avro payload envelope consumed by Server and Cloud runtimes.
9055    pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
9056        let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
9057        let mut state = self
9058            .state
9059            .lock()
9060            .map_err(|_| Error::WorkflowStatePoisoned)?;
9061
9062        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9063            return match recorded {
9064                RecordedCommand::Memo {
9065                    sequence,
9066                    entries: recorded_entries,
9067                } => {
9068                    if recorded_entries != entries {
9069                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9070                            "memo_update_mismatch",
9071                            Some(sequence),
9072                            Some(format!("{recorded_entries:?}")),
9073                            Some(format!("{entries:?}")),
9074                            "recorded memo entries differ from the current workflow update",
9075                        )));
9076                    }
9077                    state.command_cursor += 1;
9078                    Ok(())
9079                }
9080                other => Err(command_mismatch(&other, "memo upsert")),
9081            };
9082        }
9083
9084        let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
9085        state.commands.push(json!({
9086            "type": "upsert_memo",
9087            "entries": entries_envelope,
9088        }));
9089        Ok(())
9090    }
9091
9092    /// Start a named durable child on an explicit queue and await its result.
9093    ///
9094    /// The command is recorded in the parent's sequence-ordered durable command
9095    /// stream. Replay keeps a scheduled child pending without emitting another
9096    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
9097    /// values preserve the history payload codec and include both sides of the
9098    /// durable relationship; failures are returned as
9099    /// [`Error::ChildWorkflowFailed`].
9100    ///
9101    /// ```no_run
9102    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
9103    /// # fn configure(client: Client) {
9104    /// let mut worker = Worker::new(client, "parent-workers");
9105    /// worker.register_workflow("order-parent", |ctx, _input| async move {
9106    ///     let child = ctx
9107    ///         .start_child_workflow(
9108    ///             "fulfil-order",
9109    ///             ChildWorkflowOptions::new("fulfilment-workers")
9110    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
9111    ///             json!([{"order_id": "order-42"}]),
9112    ///         )
9113    ///         .await?;
9114    ///     Ok(child.result)
9115    /// });
9116    /// # }
9117    /// ```
9118    pub fn start_child_workflow<T: Serialize>(
9119        &self,
9120        workflow_type: impl Into<String>,
9121        options: ChildWorkflowOptions,
9122        args: T,
9123    ) -> ChildWorkflowCall {
9124        ChildWorkflowCall {
9125            ctx: self.clone(),
9126            workflow_type: workflow_type.into(),
9127            options,
9128            args: Some(AvroValue::from_serialize(&args)),
9129            scheduled: false,
9130            matched_pending: false,
9131            parallel_group_path: Vec::new(),
9132        }
9133    }
9134
9135    pub async fn start_child_workflow_avro_value<T: Serialize>(
9136        &self,
9137        workflow_type: impl Into<String>,
9138        options: ChildWorkflowOptions,
9139        args: T,
9140    ) -> Result<ChildWorkflowAvroResult> {
9141        let mut call = self.start_child_workflow(workflow_type, options, args);
9142        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9143    }
9144
9145    fn take_commands(&self) -> Result<Vec<Value>> {
9146        let mut state = self
9147            .state
9148            .lock()
9149            .map_err(|_| Error::WorkflowStatePoisoned)?;
9150        Ok(std::mem::take(&mut state.commands))
9151    }
9152
9153    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
9154        let mut state = self
9155            .state
9156            .lock()
9157            .map_err(|_| Error::WorkflowStatePoisoned)?;
9158
9159        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9160            return Err(command_mismatch(&recorded, "continue as new"));
9161        }
9162        if state.recorded_continue_as_new_sequence.is_some() {
9163            state.continue_as_new_consumed = true;
9164            return Ok(None);
9165        }
9166
9167        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
9168        let mut command = serde_json::Map::from_iter([
9169            ("type".to_string(), json!("continue_as_new")),
9170            ("arguments".to_string(), arguments),
9171            ("queue".to_string(), json!(state.task_queue.clone())),
9172        ]);
9173        if let Some(workflow_type) = request.options.workflow_type {
9174            command.insert("workflow_type".to_string(), json!(workflow_type));
9175        }
9176        if let Some(task_queue) = request.options.task_queue {
9177            command.insert("queue".to_string(), json!(task_queue));
9178        }
9179        Ok(Some(Value::Object(command)))
9180    }
9181
9182    fn matched_recorded_pending(&self) -> Result<bool> {
9183        let state = self
9184            .state
9185            .lock()
9186            .map_err(|_| Error::WorkflowStatePoisoned)?;
9187        Ok(state.matched_recorded_pending)
9188    }
9189
9190    fn ensure_history_consumed(&self) -> Result<()> {
9191        let state = self
9192            .state
9193            .lock()
9194            .map_err(|_| Error::WorkflowStatePoisoned)?;
9195        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
9196            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9197                "recorded_commands_unconsumed",
9198                Some(command.sequence()),
9199                Some(command.shape().to_string()),
9200                Some("workflow completion".to_string()),
9201                "workflow completed before consuming all recorded durable commands",
9202            )));
9203        }
9204        if let Some(sequence) = state
9205            .recorded_continue_as_new_sequence
9206            .filter(|_| !state.continue_as_new_consumed)
9207        {
9208            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9209                "recorded_continue_as_new_unconsumed",
9210                Some(sequence),
9211                Some("continue as new".to_string()),
9212                Some("workflow completion".to_string()),
9213                "workflow completed without consuming its recorded continue-as-new transition",
9214            )));
9215        }
9216        Ok(())
9217    }
9218}
9219
9220fn contiguous_message_stream_count(
9221    pending: &[MessageStreamMessage],
9222    cursor: u64,
9223    max_items: usize,
9224) -> usize {
9225    pending
9226        .iter()
9227        .take(max_items)
9228        .enumerate()
9229        .take_while(|(offset, message)| {
9230            u64::try_from(*offset)
9231                .ok()
9232                .and_then(|offset| cursor.checked_add(offset + 1))
9233                == Some(message.position)
9234        })
9235        .count()
9236}
9237
9238fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
9239    matches!(
9240        event.event_type.as_str(),
9241        "ActivityScheduled"
9242            | "TimerScheduled"
9243            | "ChildWorkflowScheduled"
9244            | "SignalWaitOpened"
9245            | "ConditionWaitOpened"
9246            | "SearchAttributesUpserted"
9247            | "SideEffectRecorded"
9248            | "VersionMarkerRecorded"
9249            | "MemoUpserted"
9250            | "WorkflowContinuedAsNew"
9251    )
9252}
9253
9254#[derive(Debug)]
9255struct WorkflowState {
9256    workflow_id: Option<String>,
9257    run_id: Option<String>,
9258    task_queue: String,
9259    payload_codec: String,
9260    history_events: Arc<Vec<HistoryEvent>>,
9261    history_budget: WorkflowHistoryBudget,
9262    cancel_requested: bool,
9263    resume_signal: Option<ResumeSignal>,
9264    recorded_commands: Vec<RecordedCommand>,
9265    selection_markers: Vec<SelectionMarker>,
9266    selection_marker_cursor: usize,
9267    cancelled_selection_members: Vec<SelectionCancellation>,
9268    recorded_continue_as_new_sequence: Option<u64>,
9269    continue_as_new_consumed: bool,
9270    command_cursor: usize,
9271    condition_wait_occurrence_counter: u64,
9272    matched_recorded_pending: bool,
9273    version_markers: HashMap<String, (i32, u64)>,
9274    workflow_command_identity: String,
9275    workflow_stream_command_counter: u64,
9276    commands: Vec<Value>,
9277    message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9278    message_stream_cursors: HashMap<String, u64>,
9279    message_stream_waits: HashMap<String, u64>,
9280}
9281
9282impl WorkflowState {
9283    #[cfg(test)]
9284    fn new(
9285        history: Vec<HistoryEvent>,
9286        task_queue: String,
9287        payload_codec: String,
9288        resume_signal: Option<ResumeSignal>,
9289    ) -> Result<Self> {
9290        Self::new_with_identity(
9291            history,
9292            None,
9293            None,
9294            task_queue,
9295            payload_codec,
9296            resume_signal,
9297        )
9298    }
9299
9300    fn new_with_identity(
9301        history: Vec<HistoryEvent>,
9302        workflow_id: Option<String>,
9303        run_id: Option<String>,
9304        task_queue: String,
9305        payload_codec: String,
9306        resume_signal: Option<ResumeSignal>,
9307    ) -> Result<Self> {
9308        let recorded_commands = recorded_commands(
9309            &history,
9310            &payload_codec,
9311            WorkflowIdentity {
9312                workflow_id: workflow_id.clone(),
9313                run_id: run_id.clone(),
9314            },
9315        )?;
9316        let selection_markers = recorded_selection_markers(&history)?;
9317        let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9318        let recorded_continue_as_new = history
9319            .iter()
9320            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9321            .collect::<Vec<_>>();
9322        if recorded_continue_as_new.len() > 1 {
9323            return Err(invalid_recorded_history(
9324                "duplicate_continue_as_new_transition",
9325                recorded_continue_as_new
9326                    .last()
9327                    .and_then(|event| durable_event_sequence(event))
9328                    .unwrap_or(0),
9329                "one WorkflowContinuedAsNew event",
9330                &format!(
9331                    "{} WorkflowContinuedAsNew events",
9332                    recorded_continue_as_new.len()
9333                ),
9334                "workflow history records one continue-as-new transition more than once",
9335            ));
9336        }
9337        let recorded_continue_as_new_sequence = recorded_continue_as_new
9338            .first()
9339            .map(|event| {
9340                durable_event_sequence(event).ok_or_else(|| {
9341                    Error::NonDeterministicReplay(ReplayFailure::new(
9342                        "continue_as_new_sequence_missing",
9343                        None,
9344                        Some("recorded transition sequence".to_string()),
9345                        Some("missing sequence".to_string()),
9346                        "WorkflowContinuedAsNew history is missing its recorded sequence",
9347                    ))
9348                })
9349            })
9350            .transpose()?;
9351        let mut message_stream_cursors = HashMap::new();
9352        for event in &history {
9353            if !matches!(
9354                event.event_type.as_str(),
9355                "SignalReceived" | "SignalApplied"
9356            ) || event.payload.get("signal_name").and_then(Value::as_str)
9357                != Some(MESSAGE_STREAM_SIGNAL)
9358            {
9359                continue;
9360            }
9361            let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9362            if arguments.len() != 1 {
9363                continue;
9364            }
9365            let envelope = arguments[0].clone().into_json()?;
9366            let Some(envelope) = envelope.as_object() else {
9367                continue;
9368            };
9369            if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9370            {
9371                continue;
9372            }
9373            let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9374                continue;
9375            };
9376            let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9377            else {
9378                continue;
9379            };
9380            let cursor = message_stream_cursors
9381                .entry(stream_name.to_string())
9382                .or_insert(0);
9383            *cursor = (*cursor).max(through_position);
9384        }
9385        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9386        let cancel_requested = history.iter().any(|event| {
9387            matches!(
9388                event.event_type.as_str(),
9389                "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9390            )
9391        });
9392        Ok(Self {
9393            workflow_command_identity: String::new(),
9394            workflow_stream_command_counter: 0,
9395            workflow_id,
9396            run_id,
9397            task_queue,
9398            payload_codec,
9399            history_events: Arc::new(history),
9400            history_budget: WorkflowHistoryBudget {
9401                event_count,
9402                ..WorkflowHistoryBudget::default()
9403            },
9404            cancel_requested,
9405            resume_signal,
9406            recorded_commands,
9407            selection_markers,
9408            selection_marker_cursor: 0,
9409            cancelled_selection_members,
9410            recorded_continue_as_new_sequence,
9411            continue_as_new_consumed: false,
9412            command_cursor: 0,
9413            condition_wait_occurrence_counter: 0,
9414            matched_recorded_pending: false,
9415            version_markers: HashMap::new(),
9416            commands: Vec::new(),
9417            message_stream_messages: HashMap::new(),
9418            message_stream_cursors,
9419            message_stream_waits: HashMap::new(),
9420        })
9421    }
9422}
9423
9424enum MessageStreamDelivery {
9425    Message(MessageStreamMessage),
9426    Cursor {
9427        stream_name: String,
9428        through_position: u64,
9429    },
9430}
9431
9432fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9433    if arguments.len() != 1 {
9434        return Ok(None);
9435    }
9436    let envelope = arguments
9437        .into_iter()
9438        .next()
9439        .expect("one argument was checked");
9440    let Some(envelope) = envelope.as_object() else {
9441        return Ok(None);
9442    };
9443    let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9444        return Ok(None);
9445    };
9446    if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9447        let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9448        else {
9449            return Ok(None);
9450        };
9451        return Ok(Some(MessageStreamDelivery::Cursor {
9452            stream_name: stream_name.to_string(),
9453            through_position,
9454        }));
9455    }
9456    if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9457        return Ok(None);
9458    }
9459    let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9460        return Ok(None);
9461    };
9462    let Some(position) = envelope
9463        .get("position")
9464        .and_then(Value::as_u64)
9465        .filter(|value| *value > 0)
9466    else {
9467        return Ok(None);
9468    };
9469    let Some(payload_envelope) = envelope.get("payload_envelope") else {
9470        return Ok(None);
9471    };
9472    let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9473    else {
9474        return Ok(None);
9475    };
9476    let decoded = decode_avro_value(&payload_envelope)?;
9477    let AvroValue::Array(values) = decoded else {
9478        return Ok(None);
9479    };
9480    Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9481        stream_name: stream_name.to_string(),
9482        message_id: message_id.to_string(),
9483        position,
9484        arguments: values,
9485    })))
9486}
9487
9488#[derive(Clone, Debug)]
9489enum RecordedCommand {
9490    Activity {
9491        sequence: u64,
9492        activity_type: Option<String>,
9493        options: Option<RecordedActivityOptions>,
9494        outcome: Option<ActivityOutcome>,
9495        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9496    },
9497    Timer {
9498        sequence: u64,
9499        delay_seconds: u64,
9500        fired: bool,
9501        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9502    },
9503    ChildWorkflow {
9504        sequence: u64,
9505        workflow_type: Option<String>,
9506        outcome: Option<ChildWorkflowOutcome>,
9507        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9508    },
9509    SignalWait {
9510        sequence: u64,
9511        signal_name: String,
9512        value: Option<Vec<AvroValue>>,
9513        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9514    },
9515    ConditionWait {
9516        sequence: u64,
9517        occurrence_id: String,
9518        condition_key: Option<String>,
9519        predicate_identity: String,
9520        timeout_seconds: Option<u64>,
9521        result: Option<ConditionWaitResult>,
9522        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9523    },
9524    SearchAttributes {
9525        sequence: u64,
9526        attributes: Value,
9527        attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9528    },
9529    SideEffect {
9530        sequence: u64,
9531        value: AvroValue,
9532    },
9533    VersionMarker {
9534        sequence: u64,
9535        change_id: String,
9536        version: i32,
9537    },
9538    Memo {
9539        sequence: u64,
9540        entries: AvroValue,
9541    },
9542}
9543
9544#[derive(Clone, Debug, PartialEq, Eq)]
9545struct SelectionMarker {
9546    selection_group_id: String,
9547    selection_group_base_sequence: u64,
9548    selection_group_size: usize,
9549    member_key: SelectionKey,
9550    member_index: usize,
9551    member_base_sequence: u64,
9552    member_size: usize,
9553    operation_kind: String,
9554    operation_identity: String,
9555    outcome: String,
9556    resolution_event_id: String,
9557    resolution_event_type: String,
9558}
9559
9560#[derive(Clone, Debug, PartialEq, Eq)]
9561struct SelectionCancellation {
9562    selection_group_id: String,
9563    member_key: SelectionKey,
9564    member_index: usize,
9565    member_base_sequence: u64,
9566    member_size: usize,
9567    operation_kind: String,
9568    operation_identity: String,
9569}
9570
9571fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9572    let mut markers: Vec<SelectionMarker> = Vec::new();
9573    for event in events
9574        .iter()
9575        .filter(|event| event.event_type == "SelectionResolved")
9576    {
9577        let payload = &event.payload;
9578        let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9579        let group_size = required_selection_usize(payload, "selection_group_size")?;
9580        let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9581        let member_size = required_selection_usize(payload, "member_size")?;
9582        let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9583        let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9584            invalid_recorded_history(
9585                "selection_marker_invalid",
9586                base_sequence,
9587                "non-empty selection_group_id",
9588                &payload.to_string(),
9589                "selection winner history is missing its durable group identity",
9590            )
9591        })?;
9592        let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9593        if group_id != expected_group_id {
9594            return Err(invalid_recorded_history(
9595                "selection_marker_invalid",
9596                base_sequence,
9597                &expected_group_id,
9598                &group_id,
9599                "selection winner history contains an incompatible group identity",
9600            ));
9601        }
9602        let group_end = base_sequence
9603            .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9604            .unwrap_or(u64::MAX);
9605        let member_end = member_base_sequence
9606            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9607            .unwrap_or(u64::MAX);
9608        if member_index >= group_size
9609            || member_base_sequence < base_sequence
9610            || member_end > group_end
9611        {
9612            return Err(invalid_recorded_history(
9613                "selection_marker_invalid",
9614                base_sequence,
9615                "winner member within selection group bounds",
9616                &payload.to_string(),
9617                "selection winner history contains an invalid member range",
9618            ));
9619        }
9620        let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9621            invalid_recorded_history(
9622                "selection_marker_invalid",
9623                base_sequence,
9624                "selection operation kind",
9625                &payload.to_string(),
9626                "selection winner history is missing its operation kind",
9627            )
9628        })?;
9629        if !matches!(
9630            operation_kind.as_str(),
9631            "activity" | "child" | "timer" | "signal" | "condition" | "group"
9632        ) {
9633            return Err(invalid_recorded_history(
9634                "selection_marker_invalid",
9635                base_sequence,
9636                "activity, child, timer, signal, condition, or group",
9637                &operation_kind,
9638                "selection winner history contains an unsupported operation kind",
9639            ));
9640        }
9641        let operation_identity =
9642            payload_string(payload, "operation_identity").ok_or_else(|| {
9643                invalid_recorded_history(
9644                    "selection_marker_invalid",
9645                    base_sequence,
9646                    "non-empty operation identity",
9647                    &payload.to_string(),
9648                    "selection winner history is missing its durable operation identity",
9649                )
9650            })?;
9651        let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9652            invalid_recorded_history(
9653                "selection_marker_invalid",
9654                base_sequence,
9655                "completed or failed selection outcome",
9656                &payload.to_string(),
9657                "selection winner history is missing its outcome",
9658            )
9659        })?;
9660        if !matches!(outcome.as_str(), "completed" | "failed") {
9661            return Err(invalid_recorded_history(
9662                "selection_marker_invalid",
9663                base_sequence,
9664                "completed or failed selection outcome",
9665                &outcome,
9666                "selection winner history contains an unsupported outcome",
9667            ));
9668        }
9669        let marker = SelectionMarker {
9670            selection_group_id: group_id,
9671            selection_group_base_sequence: base_sequence,
9672            selection_group_size: group_size,
9673            member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9674            member_index,
9675            member_base_sequence,
9676            member_size,
9677            operation_kind,
9678            operation_identity,
9679            outcome,
9680            resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9681                || {
9682                    invalid_recorded_history(
9683                        "selection_marker_invalid",
9684                        base_sequence,
9685                        "durable resolution_event_id",
9686                        &payload.to_string(),
9687                        "selection winner history is missing its terminal event identity",
9688                    )
9689                },
9690            )?,
9691            resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9692                || {
9693                    invalid_recorded_history(
9694                        "selection_marker_invalid",
9695                        base_sequence,
9696                        "durable resolution_event_type",
9697                        &payload.to_string(),
9698                        "selection winner history is missing its terminal event type",
9699                    )
9700                },
9701            )?,
9702        };
9703        if let Some(existing) = markers
9704            .iter()
9705            .find(|existing| existing.selection_group_id == marker.selection_group_id)
9706        {
9707            if existing != &marker {
9708                return Err(invalid_recorded_history(
9709                    "selection_marker_conflict",
9710                    base_sequence,
9711                    &format!("one winner for {}", marker.selection_group_id),
9712                    &payload.to_string(),
9713                    "selection history records conflicting winners for one durable group",
9714                ));
9715            }
9716            continue;
9717        }
9718        markers.push(marker);
9719    }
9720    Ok(markers)
9721}
9722
9723fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9724    let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9725    for event in events
9726        .iter()
9727        .filter(|event| event.event_type == "SelectionOperationCancelled")
9728    {
9729        let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9730            invalid_recorded_history(
9731                "selection_cancellation_invalid",
9732                0,
9733                "non-empty selection_group_id",
9734                &event.payload.to_string(),
9735                "selection cancellation history is missing its group identity",
9736            )
9737        })?;
9738        let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9739        let marker = SelectionCancellation {
9740            selection_group_id: group_id,
9741            member_key: selection_key_from_value(
9742                event.payload.get("member_key"),
9743                member_base_sequence,
9744            )?,
9745            member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9746            member_base_sequence,
9747            member_size: required_selection_usize(&event.payload, "member_size")?,
9748            operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9749                invalid_recorded_history(
9750                    "selection_cancellation_invalid",
9751                    member_base_sequence,
9752                    "selection operation kind",
9753                    &event.payload.to_string(),
9754                    "selection cancellation is missing its operation kind",
9755                )
9756            })?,
9757            operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9758                || {
9759                    invalid_recorded_history(
9760                        "selection_cancellation_invalid",
9761                        member_base_sequence,
9762                        "selection operation identity",
9763                        &event.payload.to_string(),
9764                        "selection cancellation is missing its operation identity",
9765                    )
9766                },
9767            )?,
9768        };
9769        if let Some(existing) = cancelled.iter().find(|recorded| {
9770            recorded.selection_group_id == marker.selection_group_id
9771                && recorded.member_base_sequence == marker.member_base_sequence
9772        }) {
9773            if existing != &marker {
9774                return Err(invalid_recorded_history(
9775                    "selection_cancellation_conflict",
9776                    member_base_sequence,
9777                    "one stable SelectionOperationCancelled marker",
9778                    &event.payload.to_string(),
9779                    "selection cancellation history contains conflicting member metadata",
9780                ));
9781            }
9782        } else {
9783            cancelled.push(marker);
9784        }
9785    }
9786    Ok(cancelled)
9787}
9788
9789fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9790    payload
9791        .get(field)
9792        .and_then(value_as_u64)
9793        .filter(|value| *value > 0)
9794        .ok_or_else(|| {
9795            invalid_recorded_history(
9796                "selection_marker_invalid",
9797                0,
9798                &format!("positive integer {field}"),
9799                &payload.to_string(),
9800                "selection history contains invalid durable identity metadata",
9801            )
9802        })
9803}
9804
9805fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9806    required_selection_usize_allow_zero(payload, field).and_then(|value| {
9807        if value > 0 {
9808            Ok(value)
9809        } else {
9810            Err(invalid_recorded_history(
9811                "selection_marker_invalid",
9812                0,
9813                &format!("positive integer {field}"),
9814                &payload.to_string(),
9815                "selection history contains invalid durable identity metadata",
9816            ))
9817        }
9818    })
9819}
9820
9821fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9822    payload
9823        .get(field)
9824        .and_then(value_as_u64)
9825        .and_then(|value| usize::try_from(value).ok())
9826        .ok_or_else(|| {
9827            invalid_recorded_history(
9828                "selection_marker_invalid",
9829                0,
9830                &format!("non-negative integer {field}"),
9831                &payload.to_string(),
9832                "selection history contains invalid durable identity metadata",
9833            )
9834        })
9835}
9836
9837#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9838struct RecordedActivityOptions {
9839    task_queue: RecordedSnapshotValue<Option<String>>,
9840    execution_mode: RecordedSnapshotValue<Option<String>>,
9841    retry_policy: ActivityRetrySnapshot,
9842}
9843
9844#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9845enum RecordedSnapshotValue<T> {
9846    /// Older history did not persist this field, so it cannot constrain replay.
9847    Unknown,
9848    Known(T),
9849}
9850
9851impl<T: PartialEq> RecordedSnapshotValue<T> {
9852    fn matches_current(&self, current: &Self) -> bool {
9853        match self {
9854            Self::Unknown => true,
9855            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9856        }
9857    }
9858}
9859
9860#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9861struct ActivityRetrySnapshot {
9862    snapshot_version: RecordedSnapshotValue<Option<u64>>,
9863    max_attempts: RecordedSnapshotValue<Option<u64>>,
9864    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9865    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9866    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9867    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9868    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9869    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9870}
9871
9872impl ActivityRetrySnapshot {
9873    fn matches_current(&self, current: &Self) -> bool {
9874        self.snapshot_version
9875            .matches_current(&current.snapshot_version)
9876            && self.max_attempts.matches_current(&current.max_attempts)
9877            && self
9878                .backoff_seconds
9879                .matches_current(&current.backoff_seconds)
9880            && self
9881                .start_to_close_timeout
9882                .matches_current(&current.start_to_close_timeout)
9883            && self
9884                .schedule_to_start_timeout
9885                .matches_current(&current.schedule_to_start_timeout)
9886            && self
9887                .schedule_to_close_timeout
9888                .matches_current(&current.schedule_to_close_timeout)
9889            && self
9890                .heartbeat_timeout
9891                .matches_current(&current.heartbeat_timeout)
9892            && self
9893                .non_retryable_error_types
9894                .matches_current(&current.non_retryable_error_types)
9895    }
9896}
9897
9898fn recorded_optional_u64(
9899    object: Option<&serde_json::Map<String, Value>>,
9900    field: &str,
9901) -> RecordedSnapshotValue<Option<u64>> {
9902    match object.and_then(|object| object.get(field)) {
9903        None => RecordedSnapshotValue::Unknown,
9904        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9905        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9906    }
9907}
9908
9909fn recorded_optional_string(
9910    object: &serde_json::Map<String, Value>,
9911    field: &str,
9912) -> RecordedSnapshotValue<Option<String>> {
9913    match object.get(field) {
9914        None => RecordedSnapshotValue::Unknown,
9915        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9916        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9917    }
9918}
9919
9920fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9921    let policy = policy.and_then(Value::as_object);
9922    let backoff_seconds = policy
9923        .and_then(|policy| policy.get("backoff_seconds"))
9924        .and_then(Value::as_array)
9925        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9926        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9927    let mut non_retryable_error_types = Vec::new();
9928    for error_type in policy
9929        .and_then(|policy| policy.get("non_retryable_error_types"))
9930        .and_then(Value::as_array)
9931        .into_iter()
9932        .flatten()
9933        .filter_map(Value::as_str)
9934        .map(str::trim)
9935        .filter(|error_type| !error_type.is_empty())
9936    {
9937        if !non_retryable_error_types
9938            .iter()
9939            .any(|recorded| recorded == error_type)
9940        {
9941            non_retryable_error_types.push(error_type.to_string());
9942        }
9943    }
9944
9945    ActivityRetrySnapshot {
9946        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9947        max_attempts: recorded_optional_u64(policy, "max_attempts"),
9948        backoff_seconds,
9949        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9950        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9951        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9952        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9953        non_retryable_error_types: if policy
9954            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9955        {
9956            RecordedSnapshotValue::Known(non_retryable_error_types)
9957        } else {
9958            RecordedSnapshotValue::Unknown
9959        },
9960    }
9961}
9962
9963fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9964    let policy = options.retry_policy.as_ref();
9965    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9966        Some(Value::Null) => None,
9967        Some(value) => value_as_u64(value),
9968        None => Some(1),
9969    };
9970    let backoff_seconds = policy
9971        .and_then(|policy| policy.get("backoff_seconds"))
9972        .and_then(Value::as_array)
9973        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9974        .unwrap_or_default();
9975    let non_retryable_error_types = policy
9976        .and_then(|policy| policy.get("non_retryable_error_types"))
9977        .and_then(Value::as_array)
9978        .into_iter()
9979        .flatten()
9980        .filter_map(Value::as_str)
9981        .map(str::to_string)
9982        .collect();
9983
9984    ActivityRetrySnapshot {
9985        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9986        max_attempts: RecordedSnapshotValue::Known(max_attempts),
9987        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9988        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9989        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9990        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9991        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9992        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9993    }
9994}
9995
9996fn activity_options_description(options: &RecordedActivityOptions) -> String {
9997    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9998}
9999
10000impl RecordedCommand {
10001    fn sequence(&self) -> u64 {
10002        match self {
10003            Self::Activity { sequence, .. }
10004            | Self::Timer { sequence, .. }
10005            | Self::ChildWorkflow { sequence, .. }
10006            | Self::SignalWait { sequence, .. }
10007            | Self::ConditionWait { sequence, .. }
10008            | Self::SearchAttributes { sequence, .. }
10009            | Self::SideEffect { sequence, .. }
10010            | Self::VersionMarker { sequence, .. }
10011            | Self::Memo { sequence, .. } => *sequence,
10012        }
10013    }
10014
10015    fn shape(&self) -> &'static str {
10016        match self {
10017            Self::Activity { .. } => "activity",
10018            Self::Timer { .. } => "timer",
10019            Self::ChildWorkflow { .. } => "child workflow",
10020            Self::SignalWait { .. } => "signal wait",
10021            Self::ConditionWait { .. } => "condition wait",
10022            Self::SearchAttributes { .. } => "search-attribute update",
10023            Self::SideEffect { .. } => "side effect",
10024            Self::VersionMarker { .. } => "version marker",
10025            Self::Memo { .. } => "memo upsert",
10026        }
10027    }
10028}
10029
10030fn ensure_version_supported(
10031    change_id: &str,
10032    version: i32,
10033    min_supported: i32,
10034    max_supported: i32,
10035    sequence: u64,
10036) -> Result<()> {
10037    if (min_supported..=max_supported).contains(&version) {
10038        return Ok(());
10039    }
10040    Err(Error::NonDeterministicReplay(ReplayFailure::new(
10041        "version_marker_incompatible_range",
10042        (sequence != 0).then_some(sequence),
10043        Some(format!("{min_supported}..={max_supported}")),
10044        Some(format!("{change_id}:{version}")),
10045        "recorded workflow version is outside the range supported by current code",
10046    )))
10047}
10048
10049#[derive(Clone, Debug)]
10050struct ResumeSignal {
10051    signal_name: String,
10052    arguments: Vec<AvroValue>,
10053}
10054
10055const MAX_PARALLEL_OPERATIONS: usize = 1000;
10056
10057fn parallel_group_prefix(kind: &str) -> &'static str {
10058    match kind {
10059        "activity" => "parallel-activities",
10060        "child" => "parallel-children",
10061        "timer" => "parallel-timers",
10062        _ => "parallel-calls",
10063    }
10064}
10065
10066fn parallel_group_entry(
10067    base_sequence: u64,
10068    size: usize,
10069    index: usize,
10070    kind: &str,
10071) -> ParallelGroupMetadata {
10072    ParallelGroupMetadata {
10073        parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
10074        parallel_group_kind: kind.to_string(),
10075        parallel_group_base_sequence: base_sequence,
10076        parallel_group_size: size,
10077        parallel_group_index: index,
10078        parallel_group_mode: None,
10079        selection_member_key: None,
10080        selection_member_index: None,
10081        selection_member_base_sequence: None,
10082        selection_member_size: None,
10083        selection_member_kind: None,
10084    }
10085}
10086
10087struct SelectionMemberMetadata {
10088    key: SelectionKey,
10089    index: usize,
10090    base_sequence: u64,
10091    size: usize,
10092    kind: String,
10093}
10094
10095fn selection_group_entry(
10096    base_sequence: u64,
10097    size: usize,
10098    index: usize,
10099    kind: &str,
10100    member: &SelectionMemberMetadata,
10101) -> ParallelGroupMetadata {
10102    ParallelGroupMetadata {
10103        parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
10104        parallel_group_kind: kind.to_string(),
10105        parallel_group_base_sequence: base_sequence,
10106        parallel_group_size: size,
10107        parallel_group_index: index,
10108        parallel_group_mode: Some("select".to_string()),
10109        selection_member_key: Some(member.key.clone()),
10110        selection_member_index: Some(member.index),
10111        selection_member_base_sequence: Some(member.base_sequence),
10112        selection_member_size: Some(member.size),
10113        selection_member_kind: Some(member.kind.clone()),
10114    }
10115}
10116
10117fn apply_parallel_group_path(
10118    command: &mut serde_json::Map<String, Value>,
10119    path: &[ParallelGroupMetadata],
10120) {
10121    let Some(inner) = path.last() else {
10122        return;
10123    };
10124    command.insert(
10125        "parallel_group_id".to_string(),
10126        json!(inner.parallel_group_id),
10127    );
10128    command.insert(
10129        "parallel_group_kind".to_string(),
10130        json!(inner.parallel_group_kind),
10131    );
10132    command.insert(
10133        "parallel_group_base_sequence".to_string(),
10134        json!(inner.parallel_group_base_sequence),
10135    );
10136    command.insert(
10137        "parallel_group_size".to_string(),
10138        json!(inner.parallel_group_size),
10139    );
10140    command.insert(
10141        "parallel_group_index".to_string(),
10142        json!(inner.parallel_group_index),
10143    );
10144    if let Some(mode) = &inner.parallel_group_mode {
10145        command.insert("parallel_group_mode".to_string(), json!(mode));
10146    }
10147    if let Some(key) = &inner.selection_member_key {
10148        command.insert("selection_member_key".to_string(), json!(key));
10149    }
10150    if let Some(index) = inner.selection_member_index {
10151        command.insert("selection_member_index".to_string(), json!(index));
10152    }
10153    if let Some(base_sequence) = inner.selection_member_base_sequence {
10154        command.insert(
10155            "selection_member_base_sequence".to_string(),
10156            json!(base_sequence),
10157        );
10158    }
10159    if let Some(size) = inner.selection_member_size {
10160        command.insert("selection_member_size".to_string(), json!(size));
10161    }
10162    if let Some(kind) = &inner.selection_member_kind {
10163        command.insert("selection_member_kind".to_string(), json!(kind));
10164    }
10165    command.insert("parallel_group_path".to_string(), json!(path));
10166}
10167
10168fn ensure_parallel_path_matches(
10169    sequence: u64,
10170    recorded: Option<&[ParallelGroupMetadata]>,
10171    expected: &[ParallelGroupMetadata],
10172) -> Result<()> {
10173    match (recorded, expected.is_empty()) {
10174        (None, true) => Ok(()),
10175        (Some(recorded), false) if recorded == expected => Ok(()),
10176        (None, false) => Err(invalid_recorded_history(
10177            "parallel_group_metadata_missing",
10178            sequence,
10179            &serde_json::to_string(expected).unwrap_or_default(),
10180            "<missing>",
10181            "recorded parallel member is missing its durable group path",
10182        )),
10183        (Some(recorded), true) => Err(invalid_recorded_history(
10184            "parallel_group_shape_mismatch",
10185            sequence,
10186            "sequential command",
10187            &serde_json::to_string(recorded).unwrap_or_default(),
10188            "recorded command belonged to a parallel group but current code schedules it sequentially",
10189        )),
10190        (Some(recorded), false) => Err(invalid_recorded_history(
10191            "parallel_group_shape_mismatch",
10192            sequence,
10193            &serde_json::to_string(recorded).unwrap_or_default(),
10194            &serde_json::to_string(expected).unwrap_or_default(),
10195            "recorded parallel-group identity or path changed during replay",
10196        )),
10197    }
10198}
10199
10200#[derive(Clone, Debug)]
10201enum ParallelShape {
10202    Leaf,
10203    Group(Vec<ParallelShape>),
10204}
10205
10206struct ParallelDescriptor {
10207    operation: ParallelOperation,
10208    offset: usize,
10209    member_path: Vec<usize>,
10210    group_path: Vec<ParallelGroupMetadata>,
10211}
10212
10213fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
10214    operations
10215        .iter()
10216        .map(|operation| match operation {
10217            ParallelOperation::Group(children) => parallel_leaf_count(children),
10218            _ => 1,
10219        })
10220        .sum()
10221}
10222
10223fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
10224    match operation {
10225        ParallelOperation::Activity { .. } => Some("activity"),
10226        ParallelOperation::ChildWorkflow { .. } => Some("child"),
10227        ParallelOperation::Timer(_) => Some("timer"),
10228        ParallelOperation::Signal(_) => Some("signal"),
10229        ParallelOperation::Condition { .. } => Some("condition"),
10230        ParallelOperation::Group(children) => parallel_group_kind(children),
10231    }
10232}
10233
10234fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
10235    let mut kind = None;
10236    for operation in operations {
10237        let Some(operation_kind) = parallel_operation_kind(operation) else {
10238            continue;
10239        };
10240        match kind {
10241            None => kind = Some(operation_kind),
10242            Some(current) if current == operation_kind => {}
10243            Some(_) => return Some("mixed"),
10244        }
10245    }
10246    kind
10247}
10248
10249fn validate_parallel_operations(
10250    operations: &[ParallelOperation],
10251    member_path: &mut Vec<usize>,
10252    root: bool,
10253) -> Result<()> {
10254    let leaves = parallel_leaf_count(operations);
10255    if leaves > MAX_PARALLEL_OPERATIONS {
10256        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10257            reason: "fan_out_limit_exceeded",
10258            member_path: member_path.clone(),
10259            message: format!(
10260                "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10261            ),
10262        }));
10263    }
10264    if !root && operations.is_empty() {
10265        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10266            reason: "nested_group_empty",
10267            member_path: member_path.clone(),
10268            message: "a nested group must contain at least one durable leaf".to_string(),
10269        }));
10270    }
10271
10272    for (index, operation) in operations.iter().enumerate() {
10273        member_path.push(index);
10274        match operation {
10275            ParallelOperation::Activity {
10276                options, arguments, ..
10277            } => {
10278                options
10279                    .validate()
10280                    .map_err(|error| Error::InvalidActivityOptions(error))?;
10281                if let Err(error) = arguments {
10282                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10283                        reason: "arguments_invalid",
10284                        member_path: member_path.clone(),
10285                        message: error.to_string(),
10286                    }));
10287                }
10288            }
10289            ParallelOperation::ChildWorkflow {
10290                options, arguments, ..
10291            } => {
10292                validate_parallel_child_options(options)?;
10293                if let Err(error) = arguments {
10294                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10295                        reason: "arguments_invalid",
10296                        member_path: member_path.clone(),
10297                        message: error.to_string(),
10298                    }));
10299                }
10300            }
10301            ParallelOperation::Timer(duration)
10302                if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10303            {
10304                return Err(Error::TimerDurationOverflow);
10305            }
10306            ParallelOperation::Timer(_) => {}
10307            ParallelOperation::Signal(signal_name) => {
10308                validate_user_signal_name(signal_name)?;
10309                if signal_name.trim().is_empty() {
10310                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10311                        reason: "signal_name_empty",
10312                        member_path: member_path.clone(),
10313                        message: "signal wait name must not be empty".to_string(),
10314                    }));
10315                }
10316            }
10317            ParallelOperation::Condition { options, .. } => {
10318                options.validate()?;
10319            }
10320            ParallelOperation::Group(children) => {
10321                validate_parallel_operations(children, member_path, false)?;
10322            }
10323        }
10324        member_path.pop();
10325    }
10326    Ok(())
10327}
10328
10329fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10330    if options.task_queue.trim().is_empty() {
10331        return Err(Error::InvalidChildWorkflowOptions(
10332            "task_queue must not be empty".to_string(),
10333        ));
10334    }
10335    for (name, value) in [
10336        (
10337            "execution_timeout_seconds",
10338            options.execution_timeout_seconds,
10339        ),
10340        ("run_timeout_seconds", options.run_timeout_seconds),
10341    ] {
10342        if value == Some(0) {
10343            return Err(Error::InvalidChildWorkflowOptions(format!(
10344                "{name} must be at least 1"
10345            )));
10346        }
10347    }
10348    if options
10349        .retry_policy
10350        .as_ref()
10351        .is_some_and(|policy| policy.max_attempts == Some(0))
10352    {
10353        return Err(Error::InvalidChildWorkflowOptions(
10354            "retry_policy.max_attempts must be at least 1".to_string(),
10355        ));
10356    }
10357    Ok(())
10358}
10359
10360fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10361    ParallelShape::Group(
10362        operations
10363            .iter()
10364            .map(|operation| match operation {
10365                ParallelOperation::Group(children) => parallel_shape(children),
10366                _ => ParallelShape::Leaf,
10367            })
10368            .collect(),
10369    )
10370}
10371
10372fn parallel_descriptors(
10373    operations: Vec<ParallelOperation>,
10374    base_sequence: u64,
10375) -> Result<Vec<ParallelDescriptor>> {
10376    let size = parallel_leaf_count(&operations);
10377    let kind = parallel_group_kind(&operations).unwrap_or("activity");
10378    let mut descriptors = Vec::with_capacity(size);
10379    let mut cursor = 0;
10380
10381    for (index, operation) in operations.into_iter().enumerate() {
10382        match operation {
10383            ParallelOperation::Group(children) => {
10384                let child_base = base_sequence
10385                    .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10386                    .ok_or(Error::TimerDurationOverflow)?;
10387                for mut descriptor in parallel_descriptors(children, child_base)? {
10388                    let outer_index = cursor + descriptor.offset;
10389                    descriptor.group_path.insert(
10390                        0,
10391                        parallel_group_entry(base_sequence, size, outer_index, kind),
10392                    );
10393                    descriptor.member_path.insert(0, index);
10394                    descriptor.offset = outer_index;
10395                    descriptors.push(descriptor);
10396                }
10397                cursor = descriptors.len();
10398            }
10399            operation => {
10400                descriptors.push(ParallelDescriptor {
10401                    operation,
10402                    offset: cursor,
10403                    member_path: vec![index],
10404                    group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10405                });
10406                cursor += 1;
10407            }
10408        }
10409    }
10410    Ok(descriptors)
10411}
10412
10413enum ParallelLeafCall {
10414    Activity(ActivityCall),
10415    ChildWorkflow(ChildWorkflowCall),
10416    Timer(TimerCall),
10417    Signal(SignalCall),
10418    Condition(ConditionWaitCall),
10419}
10420
10421fn parallel_leaf_call(
10422    ctx: &WorkflowContext,
10423    operation: ParallelOperation,
10424    parallel_group_path: Vec<ParallelGroupMetadata>,
10425) -> ParallelLeafCall {
10426    match operation {
10427        ParallelOperation::Activity {
10428            activity_type,
10429            options,
10430            arguments,
10431        } => ParallelLeafCall::Activity(ActivityCall {
10432            ctx: ctx.clone(),
10433            activity_type,
10434            options,
10435            args: Some(arguments),
10436            scheduled: false,
10437            parallel_group_path,
10438        }),
10439        ParallelOperation::ChildWorkflow {
10440            workflow_type,
10441            options,
10442            arguments,
10443        } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10444            ctx: ctx.clone(),
10445            workflow_type,
10446            options,
10447            args: Some(arguments),
10448            scheduled: false,
10449            matched_pending: false,
10450            parallel_group_path,
10451        }),
10452        ParallelOperation::Timer(duration) => {
10453            let delay_seconds = duration
10454                .as_secs()
10455                .checked_add(u64::from(duration.subsec_nanos() > 0));
10456            ParallelLeafCall::Timer(TimerCall {
10457                ctx: ctx.clone(),
10458                delay_seconds,
10459                scheduled: false,
10460                matched_pending: false,
10461                parallel_group_path,
10462            })
10463        }
10464        ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10465            ctx: ctx.clone(),
10466            signal_name,
10467            runtime_reserved_allowed: false,
10468            opened_wait: false,
10469            matched_pending: false,
10470            parallel_group_path,
10471        }),
10472        ParallelOperation::Condition { options, predicate } => {
10473            ParallelLeafCall::Condition(ConditionWaitCall {
10474                ctx: ctx.clone(),
10475                options,
10476                predicate,
10477                occurrence_id: None,
10478                opened_wait: false,
10479                parallel_group_path,
10480            })
10481        }
10482        ParallelOperation::Group(_) => {
10483            unreachable!("parallel descriptors contain only durable leaves")
10484        }
10485    }
10486}
10487
10488impl ParallelLeafCall {
10489    fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10490        match self {
10491            Self::Activity(call) => Pin::new(call)
10492                .poll_avro_value(cx)
10493                .map_ok(ParallelAvroResult::Activity),
10494            Self::ChildWorkflow(call) => Pin::new(call)
10495                .poll_avro_value(cx)
10496                .map_ok(ParallelAvroResult::ChildWorkflow),
10497            Self::Timer(call) => Pin::new(call)
10498                .poll(cx)
10499                .map_ok(|()| ParallelAvroResult::Timer),
10500            Self::Signal(call) => Pin::new(call)
10501                .poll_avro_value(cx)
10502                .map_ok(ParallelAvroResult::Signal),
10503            Self::Condition(call) => Pin::new(call)
10504                .poll(cx)
10505                .map_ok(ParallelAvroResult::Condition),
10506        }
10507    }
10508}
10509
10510struct ParallelLeaf {
10511    call: ParallelLeafCall,
10512    member_path: Vec<usize>,
10513    group_path: Vec<ParallelGroupMetadata>,
10514    result: Option<ParallelAvroResult>,
10515}
10516
10517/// Future returned by [`WorkflowContext::parallel`].
10518pub struct ParallelCall {
10519    ctx: WorkflowContext,
10520    operations: Option<Vec<ParallelOperation>>,
10521    shape: Option<ParallelShape>,
10522    leaves: Vec<ParallelLeaf>,
10523}
10524
10525impl ParallelCall {
10526    fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10527        Self {
10528            ctx,
10529            operations: Some(operations),
10530            shape: None,
10531            leaves: Vec::new(),
10532        }
10533    }
10534
10535    fn initialize(&mut self) -> Result<()> {
10536        let operations = self.operations.take().unwrap_or_default();
10537        validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10538        self.shape = Some(parallel_shape(&operations));
10539        if operations.is_empty() {
10540            return Ok(());
10541        }
10542
10543        let base_sequence = {
10544            let state = self
10545                .ctx
10546                .state
10547                .lock()
10548                .map_err(|_| Error::WorkflowStatePoisoned)?;
10549            if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10550                recorded.sequence()
10551            } else {
10552                let last = state
10553                    .recorded_commands
10554                    .last()
10555                    .map(RecordedCommand::sequence)
10556                    .unwrap_or(0);
10557                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10558                    .and_then(|sequence| sequence.checked_add(1))
10559                    .ok_or_else(|| {
10560                        Error::InvalidParallelGroup(ParallelGroupError {
10561                            reason: "sequence_overflow",
10562                            member_path: Vec::new(),
10563                            message: "parallel group sequence identity overflowed u64".to_string(),
10564                        })
10565                    })?
10566            }
10567        };
10568
10569        self.leaves = parallel_descriptors(operations, base_sequence)?
10570            .into_iter()
10571            .map(|descriptor| {
10572                let call = parallel_leaf_call(
10573                    &self.ctx,
10574                    descriptor.operation,
10575                    descriptor.group_path.clone(),
10576                );
10577                ParallelLeaf {
10578                    call,
10579                    member_path: descriptor.member_path,
10580                    group_path: descriptor.group_path,
10581                    result: None,
10582                }
10583            })
10584            .collect();
10585        Ok(())
10586    }
10587
10588    fn poll_avro_value(
10589        mut self: Pin<&mut Self>,
10590        cx: &mut TaskContext<'_>,
10591    ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10592        if self.operations.is_some() {
10593            if let Err(error) = self.initialize() {
10594                return Poll::Ready(Err(error));
10595            }
10596        }
10597        if self.leaves.is_empty() {
10598            return Poll::Ready(Ok(Vec::new()));
10599        }
10600
10601        let mut failures = Vec::new();
10602        let mut pending = false;
10603        for (index, leaf) in self.leaves.iter_mut().enumerate() {
10604            if leaf.result.is_some() {
10605                continue;
10606            }
10607            match leaf.call.poll_avro_value(cx) {
10608                Poll::Ready(Ok(result)) => leaf.result = Some(result),
10609                Poll::Ready(Err(error)) => failures.push((index, error)),
10610                Poll::Pending => pending = true,
10611            }
10612        }
10613
10614        if !failures.is_empty() {
10615            if let Some(position) = failures
10616                .iter()
10617                .position(|(_, error)| workflow_task_integrity_error(error))
10618            {
10619                return Poll::Ready(Err(failures.remove(position).1));
10620            }
10621            failures.sort_by_key(|(index, _)| *index);
10622            let (failed_index, cause) = failures.remove(0);
10623            let failed = &self.leaves[failed_index];
10624            let completed = self
10625                .leaves
10626                .iter()
10627                .filter_map(|leaf| {
10628                    leaf.result
10629                        .clone()
10630                        .and_then(|result| result.into_json_result().ok())
10631                        .map(|result| ParallelCompletion {
10632                            member_path: leaf.member_path.clone(),
10633                            result,
10634                        })
10635                })
10636                .collect();
10637            let group_id = failed
10638                .group_path
10639                .first()
10640                .map(|entry| entry.parallel_group_id.clone())
10641                .unwrap_or_default();
10642            return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10643                group_id,
10644                member_path: failed.member_path.clone(),
10645                group_path: failed.group_path.clone(),
10646                completed,
10647                cause: Box::new(cause),
10648            })));
10649        }
10650        if pending {
10651            return Poll::Pending;
10652        }
10653
10654        let mut flat_results = self
10655            .leaves
10656            .iter_mut()
10657            .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10658            .collect::<Vec<_>>()
10659            .into_iter();
10660        let results = parallel_results_for_shape(
10661            self.shape.as_ref().expect("initialized parallel shape"),
10662            &mut flat_results,
10663        );
10664        Poll::Ready(Ok(match results {
10665            ParallelAvroResult::Group(results) => results,
10666            ParallelAvroResult::Activity(_)
10667            | ParallelAvroResult::ChildWorkflow(_)
10668            | ParallelAvroResult::Timer
10669            | ParallelAvroResult::Signal(_)
10670            | ParallelAvroResult::Condition(_) => {
10671                unreachable!("root parallel shape is a group")
10672            }
10673        }))
10674    }
10675}
10676
10677fn parallel_results_for_shape(
10678    shape: &ParallelShape,
10679    flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10680) -> ParallelAvroResult {
10681    match shape {
10682        ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10683        ParallelShape::Group(children) => ParallelAvroResult::Group(
10684            children
10685                .iter()
10686                .map(|child| parallel_results_for_shape(child, flat_results))
10687                .collect(),
10688        ),
10689    }
10690}
10691
10692impl Future for ParallelCall {
10693    type Output = Result<Vec<ParallelResult>>;
10694
10695    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10696        self.poll_avro_value(cx)
10697            .map_ok(|results| {
10698                results
10699                    .into_iter()
10700                    .map(ParallelAvroResult::into_json_result)
10701                    .collect::<Result<Vec<_>>>()
10702            })
10703            .map_ok(|result| result)
10704            .flatten_result()
10705    }
10706}
10707
10708#[derive(Clone, Debug)]
10709struct SelectionMemberPlan {
10710    key: SelectionKey,
10711    index: usize,
10712    base_sequence: u64,
10713    size: usize,
10714    kind: String,
10715    shape: ParallelShape,
10716    leaf_start: usize,
10717}
10718
10719fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10720    match operation {
10721        ParallelOperation::Activity { .. } => "activity",
10722        ParallelOperation::ChildWorkflow { .. } => "child",
10723        ParallelOperation::Timer(_) => "timer",
10724        ParallelOperation::Signal(_) => "signal",
10725        ParallelOperation::Condition { .. } => "condition",
10726        ParallelOperation::Group(_) => "group",
10727    }
10728}
10729
10730fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10731    match operation {
10732        ParallelOperation::Group(children) => parallel_shape(children),
10733        _ => ParallelShape::Leaf,
10734    }
10735}
10736
10737fn selection_descriptors(
10738    operations: Vec<(SelectionKey, ParallelOperation)>,
10739    base_sequence: u64,
10740) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10741    if operations.is_empty() {
10742        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10743            reason: "selection_empty",
10744            member_path: Vec::new(),
10745            message: "durable selection requires at least one operation".to_string(),
10746        }));
10747    }
10748    let operation_refs = operations
10749        .iter()
10750        .map(|(_, operation)| operation)
10751        .collect::<Vec<_>>();
10752    let total_size = operation_refs
10753        .iter()
10754        .map(|operation| match operation {
10755            ParallelOperation::Group(children) => parallel_leaf_count(children),
10756            _ => 1,
10757        })
10758        .sum::<usize>();
10759    if total_size > MAX_PARALLEL_OPERATIONS {
10760        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10761            reason: "fan_out_limit_exceeded",
10762            member_path: Vec::new(),
10763            message: format!(
10764                "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10765            ),
10766        }));
10767    }
10768    let group_kind = {
10769        let mut kind = None;
10770        for operation in &operation_refs {
10771            let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10772            match kind {
10773                None => kind = Some(operation_kind),
10774                Some(current) if current == operation_kind => {}
10775                Some(_) => {
10776                    kind = Some("mixed");
10777                    break;
10778                }
10779            }
10780        }
10781        kind.unwrap_or("mixed")
10782    };
10783
10784    let mut descriptors = Vec::with_capacity(total_size);
10785    let mut members = Vec::with_capacity(operations.len());
10786    let mut cursor = 0usize;
10787    let mut seen_keys: Vec<SelectionKey> = Vec::new();
10788    for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10789        if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10790            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10791                reason: "selection_key_invalid",
10792                member_path: vec![member_index],
10793                message: "selection member keys must be non-empty strings or non-negative integers"
10794                    .to_string(),
10795            }));
10796        }
10797        if seen_keys.contains(&key) {
10798            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10799                reason: "selection_key_duplicate",
10800                member_path: vec![member_index],
10801                message: format!("selection member key {key:?} is duplicated"),
10802            }));
10803        }
10804        seen_keys.push(key.clone());
10805        let member_size = match &operation {
10806            ParallelOperation::Group(children) => parallel_leaf_count(children),
10807            _ => 1,
10808        };
10809        if member_size == 0 {
10810            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10811                reason: "selection_member_empty",
10812                member_path: vec![member_index],
10813                message: "a selection member must contain at least one durable leaf".to_string(),
10814            }));
10815        }
10816        let member_base = base_sequence
10817            .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10818            .ok_or(Error::TimerDurationOverflow)?;
10819        let member_kind = selection_operation_kind(&operation).to_string();
10820        let member_shape = selection_operation_shape(&operation);
10821        let leaf_start = descriptors.len();
10822        match operation {
10823            ParallelOperation::Group(children) => {
10824                validate_parallel_operations(&children, &mut vec![member_index], false)?;
10825                for mut descriptor in parallel_descriptors(children, member_base)? {
10826                    let flat_index = cursor + descriptor.offset;
10827                    descriptor.group_path.insert(
10828                        0,
10829                        selection_group_entry(
10830                            base_sequence,
10831                            total_size,
10832                            flat_index,
10833                            group_kind,
10834                            &SelectionMemberMetadata {
10835                                key: key.clone(),
10836                                index: member_index,
10837                                base_sequence: member_base,
10838                                size: member_size,
10839                                kind: member_kind.clone(),
10840                            },
10841                        ),
10842                    );
10843                    descriptor.member_path.insert(0, member_index);
10844                    descriptor.offset = flat_index;
10845                    descriptors.push(descriptor);
10846                }
10847            }
10848            operation => {
10849                validate_parallel_operations(
10850                    std::slice::from_ref(&operation),
10851                    &mut Vec::new(),
10852                    true,
10853                )?;
10854                descriptors.push(ParallelDescriptor {
10855                    operation,
10856                    offset: cursor,
10857                    member_path: vec![member_index],
10858                    group_path: vec![selection_group_entry(
10859                        base_sequence,
10860                        total_size,
10861                        cursor,
10862                        group_kind,
10863                        &SelectionMemberMetadata {
10864                            key: key.clone(),
10865                            index: member_index,
10866                            base_sequence: member_base,
10867                            size: member_size,
10868                            kind: member_kind.clone(),
10869                        },
10870                    )],
10871                });
10872            }
10873        }
10874        members.push(SelectionMemberPlan {
10875            key,
10876            index: member_index,
10877            base_sequence: member_base,
10878            size: member_size,
10879            kind: member_kind,
10880            shape: member_shape,
10881            leaf_start,
10882        });
10883        cursor += member_size;
10884    }
10885    Ok((descriptors, members))
10886}
10887
10888struct SelectionLeaf {
10889    call: ParallelLeafCall,
10890    outcome: Option<Result<ParallelAvroResult>>,
10891}
10892
10893/// Stable reference to one member of a durable selection group.
10894#[derive(Clone)]
10895pub struct DurableOperationHandle {
10896    ctx: WorkflowContext,
10897    pub key: SelectionKey,
10898    pub index: usize,
10899    pub kind: String,
10900    pub identity: String,
10901    pub base_sequence: u64,
10902    pub size: usize,
10903    pub selection_group_id: String,
10904    shape: ParallelShape,
10905}
10906
10907impl std::fmt::Debug for DurableOperationHandle {
10908    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10909        formatter
10910            .debug_struct("DurableOperationHandle")
10911            .field("key", &self.key)
10912            .field("index", &self.index)
10913            .field("kind", &self.kind)
10914            .field("identity", &self.identity)
10915            .field("base_sequence", &self.base_sequence)
10916            .field("size", &self.size)
10917            .field("selection_group_id", &self.selection_group_id)
10918            .finish()
10919    }
10920}
10921
10922impl DurableOperationHandle {
10923    /// Await this member after another member has already won the selection.
10924    pub fn await_result(&self) -> DurableOperationAwaitCall {
10925        DurableOperationAwaitCall {
10926            handle: self.clone(),
10927        }
10928    }
10929
10930    /// Request cancellation without affecting siblings. The future resolves
10931    /// to unit; only `SelectionOperationCancelled` history proves cancellation
10932    /// beat a concurrently committed terminal result.
10933    pub fn cancel(&self) -> CancelDurableOperationCall {
10934        CancelDurableOperationCall {
10935            handle: self.clone(),
10936            emitted: false,
10937        }
10938    }
10939}
10940
10941/// The one winner committed for a durable selection group.
10942#[derive(Debug)]
10943pub struct SelectionResult {
10944    pub key: SelectionKey,
10945    pub index: usize,
10946    pub kind: String,
10947    pub identity: String,
10948    pub value: Option<ParallelResult>,
10949    pub failure: Option<Error>,
10950    pub winner: DurableOperationHandle,
10951    pub handles: Vec<DurableOperationHandle>,
10952}
10953
10954impl SelectionResult {
10955    pub fn succeeded(&self) -> bool {
10956        self.failure.is_none()
10957    }
10958
10959    pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10960        self.handles.iter().find(|handle| &handle.key == key)
10961    }
10962
10963    pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10964        self.handles
10965            .iter()
10966            .filter(|handle| handle.index != self.index)
10967            .collect()
10968    }
10969
10970    pub fn into_result(self) -> Result<ParallelResult> {
10971        match (self.value, self.failure) {
10972            (Some(value), None) => Ok(value),
10973            (_, Some(error)) => Err(error),
10974            _ => Err(Error::WorkerLoop(
10975                "selection result contained neither a value nor a failure".to_string(),
10976            )),
10977        }
10978    }
10979}
10980
10981/// Future returned by [`WorkflowContext::select`].
10982pub struct SelectCall {
10983    ctx: WorkflowContext,
10984    operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10985    members: Vec<SelectionMemberPlan>,
10986    leaves: Vec<SelectionLeaf>,
10987    group_id: Option<String>,
10988}
10989
10990impl SelectCall {
10991    fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10992        Self {
10993            ctx,
10994            operations: Some(operations),
10995            members: Vec::new(),
10996            leaves: Vec::new(),
10997            group_id: None,
10998        }
10999    }
11000
11001    fn initialize(&mut self) -> Result<()> {
11002        let operations = self.operations.take().unwrap_or_default();
11003        let base_sequence = {
11004            let state = self
11005                .ctx
11006                .state
11007                .lock()
11008                .map_err(|_| Error::WorkflowStatePoisoned)?;
11009            if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
11010                marker.selection_group_base_sequence
11011            } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
11012                recorded.sequence()
11013            } else {
11014                let last = state
11015                    .recorded_commands
11016                    .last()
11017                    .map(RecordedCommand::sequence)
11018                    .unwrap_or(0);
11019                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
11020                    .and_then(|sequence| sequence.checked_add(1))
11021                    .ok_or(Error::TimerDurationOverflow)?
11022            }
11023        };
11024        let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
11025        let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
11026        self.leaves = descriptors
11027            .into_iter()
11028            .map(|descriptor| SelectionLeaf {
11029                call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
11030                outcome: None,
11031            })
11032            .collect();
11033        self.members = members;
11034        self.group_id = Some(group_id);
11035        Ok(())
11036    }
11037}
11038
11039impl Future for SelectCall {
11040    type Output = Result<SelectionResult>;
11041
11042    fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11043        if self.operations.is_some() {
11044            if let Err(error) = self.initialize() {
11045                return Poll::Ready(Err(error));
11046            }
11047        }
11048
11049        for leaf in &mut self.leaves {
11050            if leaf.outcome.is_some() {
11051                continue;
11052            }
11053            if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
11054                if outcome
11055                    .as_ref()
11056                    .err()
11057                    .is_some_and(workflow_task_integrity_error)
11058                {
11059                    return Poll::Ready(outcome.map(|_| unreachable!()));
11060                }
11061                leaf.outcome = Some(outcome);
11062            }
11063        }
11064
11065        let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
11066        let selection_member_range = self
11067            .members
11068            .first()
11069            .map(|member| member.base_sequence)
11070            .zip(self.leaves.len().try_into().ok())
11071            .map(|(base_sequence, size): (u64, u64)| {
11072                base_sequence..base_sequence.saturating_add(size)
11073            });
11074        let marker = {
11075            let mut state = match self.ctx.state.lock() {
11076                Ok(state) => state,
11077                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11078            };
11079            let marker = state
11080                .selection_markers
11081                .get(state.selection_marker_cursor)
11082                .cloned();
11083            if marker.is_none()
11084                && all_members_terminal
11085                && selection_member_range.as_ref().is_some_and(|member_range| {
11086                    state
11087                        .recorded_commands
11088                        .iter()
11089                        .any(|command| member_range.contains(&command.sequence()))
11090                })
11091            {
11092                // Terminal member history can be committed before the server's
11093                // SelectionResolved marker becomes visible to this task. That
11094                // marker is the durable barrier the selection is still waiting
11095                // on, so an otherwise commandless replay remains legitimately
11096                // pending instead of failing as an untracked yield.
11097                state.matched_recorded_pending = true;
11098            }
11099            marker
11100        };
11101        let Some(marker) = marker else {
11102            return Poll::Pending;
11103        };
11104        if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
11105            || marker.selection_group_size != self.leaves.len()
11106            || self.members.first().map(|member| member.base_sequence)
11107                != Some(marker.selection_group_base_sequence)
11108        {
11109            return Poll::Ready(Err(invalid_recorded_history(
11110                "selection_group_shape_mismatch",
11111                marker.selection_group_base_sequence,
11112                self.group_id
11113                    .as_deref()
11114                    .unwrap_or("initialized selection group"),
11115                &marker.selection_group_id,
11116                "recorded selection group differs from current workflow code",
11117            )));
11118        }
11119        let Some(member_position) = self.members.iter().position(|member| {
11120            member.key == marker.member_key
11121                && member.index == marker.member_index
11122                && member.base_sequence == marker.member_base_sequence
11123                && member.size == marker.member_size
11124                && member.kind == marker.operation_kind
11125        }) else {
11126            return Poll::Ready(Err(invalid_recorded_history(
11127                "selection_member_shape_mismatch",
11128                marker.member_base_sequence,
11129                "winner member matching current workflow code",
11130                &format!("{:?}", marker.member_key),
11131                "recorded selection winner differs from the authored member identity",
11132            )));
11133        };
11134        let member = self.members[member_position].clone();
11135        let (handles, resolution_sequence) = {
11136            let mut state = match self.ctx.state.lock() {
11137                Ok(state) => state,
11138                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11139            };
11140            let identities = self
11141                .members
11142                .iter()
11143                .map(|candidate| {
11144                    selection_operation_identity(
11145                        &state,
11146                        &candidate.kind,
11147                        candidate.base_sequence,
11148                        candidate.size,
11149                    )
11150                })
11151                .collect::<Vec<_>>();
11152            if let Some((position, missing)) = identities
11153                .iter()
11154                .enumerate()
11155                .find(|(_, identity)| identity.is_empty())
11156                .map(|(position, identity)| (position, identity.clone()))
11157            {
11158                let candidate = &self.members[position];
11159                return Poll::Ready(Err(invalid_recorded_history(
11160                    "selection_operation_identity_missing",
11161                    candidate.base_sequence,
11162                    &format!(
11163                        "durable {} resource identity from scheduled/open history",
11164                        candidate.kind
11165                    ),
11166                    &missing,
11167                    "selection member history is missing its canonical durable identity",
11168                )));
11169            }
11170            let expected_winner_identity = &identities[member_position];
11171            let resolution_sequence = match validated_selection_resolution_sequence(
11172                &state,
11173                &marker,
11174                &member,
11175                expected_winner_identity,
11176            ) {
11177                Ok(sequence) => sequence,
11178                Err(error) => return Poll::Ready(Err(error)),
11179            };
11180            let handles = self
11181                .members
11182                .iter()
11183                .zip(identities)
11184                .map(|(member, identity)| DurableOperationHandle {
11185                    ctx: self.ctx.clone(),
11186                    key: member.key.clone(),
11187                    index: member.index,
11188                    kind: member.kind.clone(),
11189                    identity,
11190                    base_sequence: member.base_sequence,
11191                    size: member.size,
11192                    selection_group_id: marker.selection_group_id.clone(),
11193                    shape: member.shape.clone(),
11194                })
11195                .collect::<Vec<_>>();
11196            if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
11197                return Poll::Ready(Err(error));
11198            }
11199            state.selection_marker_cursor += 1;
11200            (handles, resolution_sequence)
11201        };
11202
11203        let mut winner_failure = None;
11204        let mut flat_results = Vec::with_capacity(member.size);
11205        if marker.outcome == "failed" {
11206            let resolution_offset = match resolution_sequence
11207                .checked_sub(member.base_sequence)
11208                .and_then(|offset| usize::try_from(offset).ok())
11209            {
11210                Some(offset) if offset < member.size => offset,
11211                _ => {
11212                    return Poll::Ready(Err(invalid_recorded_history(
11213                        "selection_resolution_event_mismatch",
11214                        member.base_sequence,
11215                        "failure event within selected member bounds",
11216                        &resolution_sequence.to_string(),
11217                        "selection failure event is outside the authored member",
11218                    )))
11219                }
11220            };
11221            let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
11222            match leaf.outcome.take() {
11223                Some(Err(error)) => winner_failure = Some(error),
11224                _ => {
11225                    return Poll::Ready(Err(invalid_recorded_history(
11226                        "selection_winner_outcome_mismatch",
11227                        member.base_sequence,
11228                        "exact failed terminal history referenced by SelectionResolved",
11229                        "missing or successful resolution event",
11230                        "selection winner marker disagrees with terminal operation history",
11231                    )))
11232                }
11233            }
11234        } else {
11235            for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
11236                match leaf.outcome.take() {
11237                    Some(Ok(result)) => flat_results.push(result),
11238                    Some(Err(_)) => {
11239                        return Poll::Ready(Err(invalid_recorded_history(
11240                            "selection_winner_outcome_mismatch",
11241                            member.base_sequence,
11242                            "fully completed nested selection member",
11243                            "failed durable leaf",
11244                            "completed selection winner contains a failed leaf",
11245                        )))
11246                    }
11247                    None => {
11248                        return Poll::Ready(Err(invalid_recorded_history(
11249                            "selection_winner_unresolved",
11250                            member.base_sequence,
11251                            "terminal history for every completed winner leaf",
11252                            "pending member history",
11253                            "completed SelectionResolved member has an unfinished durable barrier",
11254                        )))
11255                    }
11256                }
11257            }
11258        }
11259        let value = if winner_failure.is_none() {
11260            let mut flat_results = flat_results.into_iter();
11261            let value = parallel_results_for_shape(&member.shape, &mut flat_results);
11262            match value.into_json_result() {
11263                Ok(value) => Some(value),
11264                Err(error) => return Poll::Ready(Err(error)),
11265            }
11266        } else {
11267            None
11268        };
11269        let winner = handles[member_position].clone();
11270        Poll::Ready(Ok(SelectionResult {
11271            key: winner.key.clone(),
11272            index: winner.index,
11273            kind: winner.kind.clone(),
11274            identity: winner.identity.clone(),
11275            value,
11276            failure: winner_failure,
11277            winner,
11278            handles,
11279        }))
11280    }
11281}
11282
11283fn selection_operation_identity(
11284    state: &WorkflowState,
11285    kind: &str,
11286    base_sequence: u64,
11287    size: usize,
11288) -> String {
11289    if kind == "group" {
11290        return format!("group:{base_sequence}:{size}");
11291    }
11292    let fields: &[&str] = match kind {
11293        "activity" => &["activity_execution_id"],
11294        "child" => &["child_workflow_run_id"],
11295        "timer" => &["timer_id"],
11296        "signal" => &["signal_wait_id"],
11297        "condition" => &["condition_wait_id"],
11298        _ => &[],
11299    };
11300    for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11301        for event in state
11302            .history_events
11303            .iter()
11304            .filter(|event| durable_event_sequence(event) == Some(sequence))
11305        {
11306            for field in fields {
11307                if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11308                    if !identity.is_empty() {
11309                        return identity.to_string();
11310                    }
11311                }
11312            }
11313        }
11314    }
11315    String::new()
11316}
11317
11318fn validated_selection_resolution_sequence(
11319    state: &WorkflowState,
11320    marker: &SelectionMarker,
11321    member: &SelectionMemberPlan,
11322    expected_identity: &str,
11323) -> Result<u64> {
11324    if expected_identity.is_empty() {
11325        return Err(invalid_recorded_history(
11326            "selection_operation_identity_missing",
11327            member.base_sequence,
11328            &format!(
11329                "durable {} resource identity from scheduled/open history",
11330                member.kind
11331            ),
11332            "missing operation identity",
11333            "selection member history is missing its canonical durable identity",
11334        ));
11335    }
11336    if marker.operation_identity != expected_identity {
11337        return Err(invalid_recorded_history(
11338            "selection_operation_identity_mismatch",
11339            member.base_sequence,
11340            expected_identity,
11341            &marker.operation_identity,
11342            "selection winner identity does not match durable scheduled/open history",
11343        ));
11344    }
11345
11346    let failure_types = [
11347        "ActivityFailed",
11348        "ActivityCancelled",
11349        "ActivityTimedOut",
11350        "ChildRunFailed",
11351        "ChildRunCancelled",
11352        "ChildRunTerminated",
11353    ];
11354    let success_types = [
11355        "ActivityCompleted",
11356        "ChildRunCompleted",
11357        "TimerFired",
11358        "SignalApplied",
11359        "ConditionWaitSatisfied",
11360        "ConditionWaitTimedOut",
11361    ];
11362    let terminal_types: &[&str] = if marker.outcome == "failed" {
11363        &failure_types
11364    } else {
11365        &success_types
11366    };
11367    let mut candidates = Vec::new();
11368    for event in state.history_events.iter() {
11369        let Some(sequence) = durable_event_sequence(event) else {
11370            continue;
11371        };
11372        if sequence < member.base_sequence
11373            || sequence >= member.base_sequence.saturating_add(member.size as u64)
11374            || !terminal_types.contains(&event.event_type.as_str())
11375        {
11376            continue;
11377        }
11378        let event_id = event
11379            .raw
11380            .get("id")
11381            .or_else(|| event.raw.get("event_id"))
11382            .and_then(Value::as_str)
11383            .filter(|value| !value.is_empty())
11384            .ok_or_else(|| {
11385                invalid_recorded_history(
11386                    "selection_resolution_event_id_missing",
11387                    member.base_sequence,
11388                    "terminal selection history with a durable event id",
11389                    &event.payload.to_string(),
11390                    "selection terminal history cannot be bound to its winner marker",
11391                )
11392            })?;
11393        candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11394    }
11395    let resolution = if marker.outcome == "failed" {
11396        candidates.first()
11397    } else {
11398        candidates.last()
11399    };
11400    let Some((event_id, event_type, sequence)) = resolution else {
11401        return Err(invalid_recorded_history(
11402            "selection_resolution_event_missing",
11403            member.base_sequence,
11404            "terminal history for the selected member",
11405            &format!("{:?}", marker.member_key),
11406            "selection winner marker has no matching durable terminal event",
11407        ));
11408    };
11409    if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11410        return Err(invalid_recorded_history(
11411            "selection_resolution_event_mismatch",
11412            member.base_sequence,
11413            &format!("{event_type}:{event_id}"),
11414            &format!(
11415                "{}:{}",
11416                marker.resolution_event_type, marker.resolution_event_id
11417            ),
11418            "selection winner marker does not reference the event that made its member terminal",
11419        ));
11420    }
11421    Ok(*sequence)
11422}
11423
11424fn recorded_selection_member_outcome(
11425    state: &WorkflowState,
11426    handle: &DurableOperationHandle,
11427) -> Result<Option<ParallelResult>> {
11428    for event in state.history_events.iter() {
11429        let Some(sequence) = durable_event_sequence(event) else {
11430            continue;
11431        };
11432        if sequence < handle.base_sequence
11433            || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11434            || !matches!(
11435                event.event_type.as_str(),
11436                "ActivityFailed"
11437                    | "ActivityCancelled"
11438                    | "ActivityTimedOut"
11439                    | "ChildRunFailed"
11440                    | "ChildRunCancelled"
11441                    | "ChildRunTerminated"
11442            )
11443        {
11444            continue;
11445        }
11446        let Some(command) = state
11447            .recorded_commands
11448            .iter()
11449            .find(|command| command.sequence() == sequence)
11450        else {
11451            continue;
11452        };
11453        match command {
11454            RecordedCommand::Activity {
11455                outcome: Some(Err(failure)),
11456                ..
11457            } => return Err(Error::ActivityFailed(failure.clone())),
11458            RecordedCommand::ChildWorkflow {
11459                outcome: Some(Err(failure)),
11460                ..
11461            } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11462            _ => {}
11463        }
11464    }
11465
11466    let mut results = Vec::with_capacity(handle.size);
11467    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11468        let Some(command) = state
11469            .recorded_commands
11470            .iter()
11471            .find(|command| command.sequence() == sequence)
11472        else {
11473            return Ok(None);
11474        };
11475        let result = match command {
11476            RecordedCommand::Activity { outcome, .. } => match outcome {
11477                Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11478                Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11479                None => return Ok(None),
11480            },
11481            RecordedCommand::Timer { fired, .. } => {
11482                if !fired {
11483                    return Ok(None);
11484                }
11485                ParallelAvroResult::Timer
11486            }
11487            RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11488                Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11489                Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11490                None => return Ok(None),
11491            },
11492            RecordedCommand::SignalWait { value, .. } => match value {
11493                Some(value) => ParallelAvroResult::Signal(value.clone()),
11494                None => return Ok(None),
11495            },
11496            RecordedCommand::ConditionWait { result, .. } => match result {
11497                Some(result) => ParallelAvroResult::Condition(*result),
11498                None => return Ok(None),
11499            },
11500            other => {
11501                return Err(command_mismatch(
11502                    other,
11503                    format!("selected {} member", handle.kind),
11504                ))
11505            }
11506        };
11507        results.push(result);
11508    }
11509    let mut results = results.into_iter();
11510    parallel_results_for_shape(&handle.shape, &mut results)
11511        .into_json_result()
11512        .map(Some)
11513}
11514
11515fn recorded_selection_member_is_terminal(
11516    state: &WorkflowState,
11517    handle: &DurableOperationHandle,
11518) -> bool {
11519    let mut completed = 0usize;
11520    let mut all_completed = true;
11521    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11522        let Some(command) = state
11523            .recorded_commands
11524            .iter()
11525            .find(|command| command.sequence() == sequence)
11526        else {
11527            all_completed = false;
11528            continue;
11529        };
11530        let terminal = match command {
11531            RecordedCommand::Activity {
11532                outcome: Some(Err(_)),
11533                ..
11534            }
11535            | RecordedCommand::ChildWorkflow {
11536                outcome: Some(Err(_)),
11537                ..
11538            } => return true,
11539            RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11540            RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11541            RecordedCommand::Timer { fired, .. } => *fired,
11542            RecordedCommand::SignalWait { value, .. } => value.is_some(),
11543            RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11544            RecordedCommand::SearchAttributes { .. }
11545            | RecordedCommand::SideEffect { .. }
11546            | RecordedCommand::VersionMarker { .. }
11547            | RecordedCommand::Memo { .. } => false,
11548        };
11549        if !terminal {
11550            all_completed = false;
11551            continue;
11552        }
11553        completed += 1;
11554    }
11555    all_completed && completed == handle.size
11556}
11557
11558fn selection_cancellation_for_handle(
11559    state: &WorkflowState,
11560    handle: &DurableOperationHandle,
11561) -> Result<bool> {
11562    let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11563        recorded.selection_group_id == handle.selection_group_id
11564            && recorded.member_base_sequence == handle.base_sequence
11565    }) else {
11566        return Ok(false);
11567    };
11568    validate_selection_cancellation_marker(marker, handle)?;
11569    Ok(true)
11570}
11571
11572fn validate_selection_cancellations_for_handles(
11573    state: &WorkflowState,
11574    handles: &[DurableOperationHandle],
11575) -> Result<()> {
11576    let Some(group_id) = handles
11577        .first()
11578        .map(|handle| handle.selection_group_id.as_str())
11579    else {
11580        return Ok(());
11581    };
11582    for marker in state
11583        .cancelled_selection_members
11584        .iter()
11585        .filter(|marker| marker.selection_group_id == group_id)
11586    {
11587        let Some(handle) = handles
11588            .iter()
11589            .find(|handle| handle.base_sequence == marker.member_base_sequence)
11590        else {
11591            return Err(invalid_recorded_history(
11592                "selection_cancellation_member_mismatch",
11593                marker.member_base_sequence,
11594                "SelectionOperationCancelled matching an authored selection handle",
11595                &format!("{marker:?}"),
11596                "selection cancellation member base does not name an authored member",
11597            ));
11598        };
11599        validate_selection_cancellation_marker(marker, handle)?;
11600    }
11601    Ok(())
11602}
11603
11604fn validate_selection_cancellation_marker(
11605    marker: &SelectionCancellation,
11606    handle: &DurableOperationHandle,
11607) -> Result<()> {
11608    if marker.selection_group_id != handle.selection_group_id
11609        || marker.member_key != handle.key
11610        || marker.member_index != handle.index
11611        || marker.member_base_sequence != handle.base_sequence
11612        || marker.member_size != handle.size
11613        || marker.operation_kind != handle.kind
11614        || marker.operation_identity != handle.identity
11615    {
11616        return Err(invalid_recorded_history(
11617            "selection_cancellation_member_mismatch",
11618            handle.base_sequence,
11619            "SelectionOperationCancelled matching the authored selection handle",
11620            &format!("{marker:?}"),
11621            "selection cancellation history targets different authored member metadata",
11622        ));
11623    }
11624    Ok(())
11625}
11626
11627/// Future returned by [`DurableOperationHandle::await_result`].
11628pub struct DurableOperationAwaitCall {
11629    handle: DurableOperationHandle,
11630}
11631
11632impl Future for DurableOperationAwaitCall {
11633    type Output = Result<ParallelResult>;
11634
11635    fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11636        let state = match self.handle.ctx.state.lock() {
11637            Ok(state) => state,
11638            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11639        };
11640        match selection_cancellation_for_handle(&state, &self.handle) {
11641            Err(error) => return Poll::Ready(Err(error)),
11642            Ok(false) => {}
11643            Ok(true) => {
11644                return Poll::Ready(Err(Error::DurableOperationCancelled(
11645                    DurableOperationCancelled {
11646                        selection_group_id: self.handle.selection_group_id.clone(),
11647                        member_key: self.handle.key.clone(),
11648                        member_index: self.handle.index,
11649                        operation_kind: self.handle.kind.clone(),
11650                        operation_identity: self.handle.identity.clone(),
11651                    },
11652                )));
11653            }
11654        }
11655        match recorded_selection_member_outcome(&state, &self.handle) {
11656            Ok(Some(result)) => Poll::Ready(Ok(result)),
11657            Ok(None) => Poll::Pending,
11658            Err(error) => Poll::Ready(Err(error)),
11659        }
11660    }
11661}
11662
11663/// Future returned by [`DurableOperationHandle::cancel`].
11664pub struct CancelDurableOperationCall {
11665    handle: DurableOperationHandle,
11666    emitted: bool,
11667}
11668
11669impl Future for CancelDurableOperationCall {
11670    type Output = Result<()>;
11671
11672    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11673        let ctx = self.handle.ctx.clone();
11674        let mut state = match ctx.state.lock() {
11675            Ok(state) => state,
11676            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11677        };
11678        match selection_cancellation_for_handle(&state, &self.handle) {
11679            Err(error) => return Poll::Ready(Err(error)),
11680            Ok(true) => return Poll::Ready(Ok(())),
11681            Ok(false) => {}
11682        }
11683        if recorded_selection_member_is_terminal(&state, &self.handle) {
11684            return Poll::Ready(Ok(()));
11685        }
11686        if !self.emitted {
11687            state.commands.push(json!({
11688                "type": "cancel_selection_operation",
11689                "selection_group_id": self.handle.selection_group_id,
11690                "member_key": self.handle.key,
11691                "member_index": self.handle.index,
11692                "member_base_sequence": self.handle.base_sequence,
11693                "member_size": self.handle.size,
11694                "operation_kind": self.handle.kind,
11695                "operation_identity": self.handle.identity,
11696            }));
11697            self.emitted = true;
11698        }
11699        // The cancellation command is only a request. Do not expose workflow
11700        // state after cancel until committed SelectionOperationCancelled
11701        // history proves that the request won the race with member completion.
11702        Poll::Pending
11703    }
11704}
11705
11706trait PollNestedResultExt<T> {
11707    fn flatten_result(self) -> Poll<Result<T>>;
11708}
11709
11710impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11711    fn flatten_result(self) -> Poll<Result<T>> {
11712        match self {
11713            Poll::Ready(Ok(result)) => Poll::Ready(result),
11714            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11715            Poll::Pending => Poll::Pending,
11716        }
11717    }
11718}
11719
11720struct SagaCompensation {
11721    activity_type: String,
11722    options: ActivityOptions,
11723    arguments: AvroValue,
11724    registration_order: usize,
11725}
11726
11727/// Workflow-local deterministic saga compensation helper.
11728///
11729/// Register each compensation only after its forward step succeeds. Passing
11730/// the forward `Result` to [`Saga::finish`] runs compensations sequentially in
11731/// reverse registration order after any failure, including cooperative
11732/// cancellation. Each compensation is an ordinary durable activity, so replay,
11733/// duplicate delivery, and worker restart use existing history semantics.
11734pub struct Saga {
11735    ctx: WorkflowContext,
11736    compensations: Vec<SagaCompensation>,
11737}
11738
11739impl Saga {
11740    fn new(ctx: WorkflowContext) -> Self {
11741        Self {
11742            ctx,
11743            compensations: Vec::new(),
11744        }
11745    }
11746
11747    pub fn add_compensation<T: Serialize>(
11748        &mut self,
11749        activity_type: impl Into<String>,
11750        args: T,
11751    ) -> Result<&mut Self> {
11752        self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11753    }
11754
11755    pub fn add_compensation_with_options<T: Serialize>(
11756        &mut self,
11757        activity_type: impl Into<String>,
11758        options: ActivityOptions,
11759        args: T,
11760    ) -> Result<&mut Self> {
11761        let activity_type = activity_type.into();
11762        if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11763            return Err(Error::Codec(
11764                "saga compensation activity type must be non-empty without surrounding whitespace"
11765                    .to_string(),
11766            ));
11767        }
11768        options.validate().map_err(Error::InvalidActivityOptions)?;
11769        let arguments = AvroValue::from_serialize(&args)?;
11770        let registration_order = self.compensations.len() + 1;
11771        self.compensations.push(SagaCompensation {
11772            activity_type,
11773            options,
11774            arguments,
11775            registration_order,
11776        });
11777        Ok(self)
11778    }
11779
11780    /// Compensate `initiating_failure` and return the failure that must remain.
11781    pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11782        while let Some(compensation) = self.compensations.pop() {
11783            if let Err(compensation_failure) = self
11784                .ctx
11785                .activity_with_options(
11786                    compensation.activity_type.clone(),
11787                    compensation.options,
11788                    compensation.arguments,
11789                )
11790                .await
11791            {
11792                if workflow_task_integrity_error(&compensation_failure) {
11793                    return compensation_failure;
11794                }
11795                return Error::SagaCompensationFailed(SagaCompensationFailure {
11796                    initiating_failure: Box::new(initiating_failure),
11797                    compensation_failure: Box::new(compensation_failure),
11798                    compensation_activity_type: compensation.activity_type,
11799                    compensation_registration_order: compensation.registration_order,
11800                });
11801            }
11802        }
11803        initiating_failure
11804    }
11805
11806    /// Return a successful forward value or compensate and preserve its failure.
11807    pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11808        match outcome {
11809            Ok(value) => Ok(value),
11810            Err(error) => Err(self.compensate(error).await),
11811        }
11812    }
11813}
11814
11815pub struct ActivityCall {
11816    ctx: WorkflowContext,
11817    activity_type: String,
11818    options: ActivityOptions,
11819    args: Option<Result<AvroValue>>,
11820    scheduled: bool,
11821    parallel_group_path: Vec<ParallelGroupMetadata>,
11822}
11823
11824impl ActivityCall {
11825    fn poll_avro_value(
11826        mut self: Pin<&mut Self>,
11827        _cx: &mut TaskContext<'_>,
11828    ) -> Poll<Result<AvroValue>> {
11829        let ctx = self.ctx.clone();
11830        let mut state = match ctx.state.lock() {
11831            Ok(state) => state,
11832            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11833        };
11834
11835        if self.scheduled {
11836            return Poll::Pending;
11837        }
11838
11839        let options = match self.options.validate() {
11840            Ok(options) => options,
11841            Err(error) => {
11842                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11843            }
11844        };
11845        let task_queue = options
11846            .task_queue
11847            .clone()
11848            .unwrap_or_else(|| state.task_queue.clone());
11849        let current_recorded_options = RecordedActivityOptions {
11850            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11851            // Rust schedules ordinary durable activities. The server records a
11852            // non-null mode only for a specialized execution primitive.
11853            execution_mode: RecordedSnapshotValue::Known(None),
11854            retry_policy: current_activity_retry_snapshot(&options),
11855        };
11856
11857        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11858            let sequence = recorded.sequence();
11859            match recorded {
11860                RecordedCommand::Activity {
11861                    activity_type,
11862                    options: recorded_options,
11863                    outcome,
11864                    parallel_group_path,
11865                    ..
11866                } => {
11867                    if let Err(error) = ensure_parallel_path_matches(
11868                        sequence,
11869                        parallel_group_path.as_deref(),
11870                        &self.parallel_group_path,
11871                    ) {
11872                        return Poll::Ready(Err(error));
11873                    }
11874                    if let Some(recorded_type) = activity_type {
11875                        if recorded_type != self.activity_type {
11876                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11877                                ReplayFailure::new(
11878                                    "recorded_command_detail_mismatch",
11879                                    Some(sequence),
11880                                    Some(format!("activity:{recorded_type}")),
11881                                    Some(format!("activity:{}", self.activity_type)),
11882                                    "recorded activity type differs from the current workflow command",
11883                                ),
11884                            )));
11885                        }
11886                    }
11887                    if let Some(recorded_options) = recorded_options {
11888                        if !recorded_options
11889                            .task_queue
11890                            .matches_current(&current_recorded_options.task_queue)
11891                        {
11892                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11893                                ReplayFailure::new(
11894                                    "activity_task_queue_mismatch",
11895                                    Some(sequence),
11896                                    Some(activity_options_description(&recorded_options)),
11897                                    Some(activity_options_description(&current_recorded_options)),
11898                                    "recorded activity task queue differs from the current workflow command",
11899                                ),
11900                            )));
11901                        }
11902                        if !recorded_options
11903                            .execution_mode
11904                            .matches_current(&current_recorded_options.execution_mode)
11905                        {
11906                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11907                                ReplayFailure::new(
11908                                    "activity_execution_mode_mismatch",
11909                                    Some(sequence),
11910                                    Some(activity_options_description(&recorded_options)),
11911                                    Some(activity_options_description(&current_recorded_options)),
11912                                    "recorded activity execution mode differs from the current workflow command",
11913                                ),
11914                            )));
11915                        }
11916                        if !recorded_options
11917                            .retry_policy
11918                            .matches_current(&current_recorded_options.retry_policy)
11919                        {
11920                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11921                                ReplayFailure::new(
11922                                    "activity_retry_policy_mismatch",
11923                                    Some(sequence),
11924                                    Some(activity_options_description(&recorded_options)),
11925                                    Some(activity_options_description(&current_recorded_options)),
11926                                    "recorded activity retry policy differs from the current workflow command",
11927                                ),
11928                            )));
11929                        }
11930                    }
11931                    state.command_cursor += 1;
11932                    if let Some(outcome) = outcome {
11933                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11934                    }
11935                    state.matched_recorded_pending = true;
11936                    self.scheduled = true;
11937                    return Poll::Pending;
11938                }
11939                other => {
11940                    return Poll::Ready(Err(command_mismatch(
11941                        &other,
11942                        format!("activity:{}", self.activity_type),
11943                    )));
11944                }
11945            }
11946        }
11947
11948        if !self.scheduled {
11949            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11950                Ok(args) => args,
11951                Err(error) => return Poll::Ready(Err(error)),
11952            };
11953            let arguments = normalize_avro_arguments(args);
11954            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11955                Ok(envelope) => envelope,
11956                Err(error) => return Poll::Ready(Err(error)),
11957            };
11958
11959            let mut command = serde_json::Map::from_iter([
11960                ("type".to_string(), json!("schedule_activity")),
11961                (
11962                    "activity_type".to_string(),
11963                    json!(self.activity_type.clone()),
11964                ),
11965                ("queue".to_string(), json!(task_queue)),
11966                ("arguments".to_string(), envelope),
11967            ]);
11968            for (field, value) in [
11969                ("start_to_close_timeout", options.start_to_close_timeout),
11970                (
11971                    "schedule_to_start_timeout",
11972                    options.schedule_to_start_timeout,
11973                ),
11974                (
11975                    "schedule_to_close_timeout",
11976                    options.schedule_to_close_timeout,
11977                ),
11978                ("heartbeat_timeout", options.heartbeat_timeout),
11979            ] {
11980                if let Some(value) = value {
11981                    command.insert(field.to_string(), json!(value));
11982                }
11983            }
11984            if let Some(retry_policy) = options.retry_policy {
11985                command.insert("retry_policy".to_string(), retry_policy);
11986            }
11987            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11988            state.commands.push(Value::Object(command));
11989            self.scheduled = true;
11990        }
11991
11992        Poll::Pending
11993    }
11994}
11995
11996impl Future for ActivityCall {
11997    type Output = Result<Value>;
11998
11999    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12000        match self.poll_avro_value(cx) {
12001            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
12002            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12003            Poll::Pending => Poll::Pending,
12004        }
12005    }
12006}
12007
12008/// Future returned by [`WorkflowContext::sleep`].
12009pub struct TimerCall {
12010    ctx: WorkflowContext,
12011    delay_seconds: Option<u64>,
12012    scheduled: bool,
12013    matched_pending: bool,
12014    parallel_group_path: Vec<ParallelGroupMetadata>,
12015}
12016
12017impl Future for TimerCall {
12018    type Output = Result<()>;
12019
12020    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12021        if self.matched_pending {
12022            return Poll::Pending;
12023        }
12024
12025        let ctx = self.ctx.clone();
12026        let Some(requested_delay) = self.delay_seconds else {
12027            return Poll::Ready(Err(Error::TimerDurationOverflow));
12028        };
12029        let mut state = match ctx.state.lock() {
12030            Ok(state) => state,
12031            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12032        };
12033
12034        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12035            match recorded {
12036                RecordedCommand::Timer {
12037                    sequence,
12038                    delay_seconds,
12039                    fired,
12040                    parallel_group_path,
12041                    ..
12042                } => {
12043                    if let Err(error) = ensure_parallel_path_matches(
12044                        sequence,
12045                        parallel_group_path.as_deref(),
12046                        &self.parallel_group_path,
12047                    ) {
12048                        return Poll::Ready(Err(error));
12049                    }
12050                    if delay_seconds != requested_delay {
12051                        return Poll::Ready(Err(Error::NonDeterministicReplay(
12052                            ReplayFailure::new(
12053                                "timer_delay_mismatch",
12054                                Some(sequence),
12055                                Some(format!("timer:{delay_seconds}s")),
12056                                Some(format!("timer:{requested_delay}s")),
12057                                "recorded timer delay differs from the current workflow command",
12058                            ),
12059                        )));
12060                    }
12061                    state.command_cursor += 1;
12062                    if fired {
12063                        return Poll::Ready(Ok(()));
12064                    }
12065                    state.matched_recorded_pending = true;
12066                    self.scheduled = true;
12067                    self.matched_pending = true;
12068                    return Poll::Pending;
12069                }
12070                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
12071            }
12072        }
12073
12074        if !self.scheduled {
12075            let mut command = serde_json::Map::from_iter([
12076                ("type".to_string(), json!("start_timer")),
12077                ("delay_seconds".to_string(), json!(requested_delay)),
12078            ]);
12079            apply_parallel_group_path(&mut command, &self.parallel_group_path);
12080            state.commands.push(Value::Object(command));
12081            self.scheduled = true;
12082        }
12083
12084        Poll::Pending
12085    }
12086}
12087
12088/// Future returned by [`WorkflowContext::wait_condition`].
12089pub struct ConditionWaitCall {
12090    ctx: WorkflowContext,
12091    options: ConditionWaitOptions,
12092    predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
12093    occurrence_id: Option<String>,
12094    opened_wait: bool,
12095    parallel_group_path: Vec<ParallelGroupMetadata>,
12096}
12097
12098impl Future for ConditionWaitCall {
12099    type Output = Result<ConditionWaitResult>;
12100
12101    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12102        if self.opened_wait {
12103            return Poll::Pending;
12104        }
12105
12106        let options = match self.options.validate() {
12107            Ok(options) => options,
12108            Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
12109        };
12110        let ctx = self.ctx.clone();
12111        let occurrence_id = match self.occurrence_id.as_ref() {
12112            Some(occurrence_id) => occurrence_id.clone(),
12113            None => {
12114                let mut state = match ctx.state.lock() {
12115                    Ok(state) => state,
12116                    Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12117                };
12118                let ordinal = state.condition_wait_occurrence_counter;
12119                state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
12120                    Some(next) => next,
12121                    None => {
12122                        return Poll::Ready(Err(Error::WorkerLoop(
12123                            "condition wait occurrence counter overflowed".to_string(),
12124                        )))
12125                    }
12126                };
12127                let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
12128                drop(state);
12129                self.occurrence_id = Some(occurrence_id.clone());
12130                occurrence_id
12131            }
12132        };
12133
12134        let recorded_result = {
12135            let mut state = match ctx.state.lock() {
12136                Ok(state) => state,
12137                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12138            };
12139            let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
12140                drop(state);
12141                return self.poll_new_condition(options);
12142            };
12143            if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
12144                return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
12145            }
12146
12147            let mut cursor = state.command_cursor;
12148            let mut result = None;
12149            loop {
12150                let Some(RecordedCommand::ConditionWait {
12151                    sequence,
12152                    occurrence_id: recorded_occurrence_id,
12153                    condition_key,
12154                    predicate_identity,
12155                    timeout_seconds,
12156                    result: recorded_result,
12157                    parallel_group_path,
12158                    ..
12159                }) = state.recorded_commands.get(cursor)
12160                else {
12161                    break;
12162                };
12163
12164                if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
12165                    break;
12166                }
12167                if let Err(error) = ensure_parallel_path_matches(
12168                    *sequence,
12169                    parallel_group_path.as_deref(),
12170                    &self.parallel_group_path,
12171                ) {
12172                    return Poll::Ready(Err(error));
12173                }
12174                if let Err(error) = validate_recorded_condition_wait(
12175                    *sequence,
12176                    recorded_occurrence_id,
12177                    condition_key.as_deref(),
12178                    predicate_identity,
12179                    *timeout_seconds,
12180                    &occurrence_id,
12181                    &options,
12182                ) {
12183                    return Poll::Ready(Err(error));
12184                }
12185                if result == Some(ConditionWaitResult::TimedOut) {
12186                    return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
12187                        "condition_wait_reopened_after_timeout",
12188                        Some(*sequence),
12189                        Some("timed-out condition is terminal".to_string()),
12190                        Some("another physical wait-open".to_string()),
12191                        "condition history reopened one logical wait after its durable timeout",
12192                    ))));
12193                }
12194                result = *recorded_result;
12195                cursor += 1;
12196            }
12197            state.command_cursor = cursor;
12198            result
12199        };
12200
12201        if let Some(result) = recorded_result {
12202            return Poll::Ready(Ok(result));
12203        }
12204
12205        self.poll_open_condition(options)
12206    }
12207}
12208
12209impl ConditionWaitCall {
12210    fn poll_new_condition(
12211        self: Pin<&mut Self>,
12212        options: ValidatedConditionWaitOptions,
12213    ) -> Poll<Result<ConditionWaitResult>> {
12214        self.poll_open_condition(options)
12215    }
12216
12217    fn poll_open_condition(
12218        mut self: Pin<&mut Self>,
12219        options: ValidatedConditionWaitOptions,
12220    ) -> Poll<Result<ConditionWaitResult>> {
12221        let selection_member = self
12222            .parallel_group_path
12223            .first()
12224            .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
12225        match (self.predicate)() {
12226            Ok(true) if !selection_member => {
12227                return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
12228            }
12229            Ok(_) => {}
12230            Err(error) => return Poll::Ready(Err(error)),
12231        }
12232        if options.timeout_seconds == Some(0) && !selection_member {
12233            return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
12234        }
12235
12236        let ctx = self.ctx.clone();
12237        let mut state = match ctx.state.lock() {
12238            Ok(state) => state,
12239            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12240        };
12241        let mut command = serde_json::Map::from_iter([
12242            ("type".to_string(), json!("open_condition_wait")),
12243            (
12244                "condition_wait_occurrence_id".to_string(),
12245                json!(self.occurrence_id.as_deref().unwrap_or_default()),
12246            ),
12247            ("condition_key".to_string(), json!(options.condition_key)),
12248            (
12249                "condition_definition_fingerprint".to_string(),
12250                json!(options.predicate_identity),
12251            ),
12252        ]);
12253        if let Some(timeout_seconds) = options.timeout_seconds {
12254            command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
12255        }
12256        apply_parallel_group_path(&mut command, &self.parallel_group_path);
12257        state.commands.push(Value::Object(command));
12258        drop(state);
12259        self.opened_wait = true;
12260        Poll::Pending
12261    }
12262}
12263
12264fn validate_recorded_condition_wait(
12265    sequence: u64,
12266    recorded_occurrence_id: &str,
12267    recorded_key: Option<&str>,
12268    recorded_predicate_identity: &str,
12269    recorded_timeout_seconds: Option<u64>,
12270    current_occurrence_id: &str,
12271    current: &ValidatedConditionWaitOptions,
12272) -> Result<()> {
12273    if recorded_occurrence_id != current_occurrence_id {
12274        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12275            "condition_wait_occurrence_mismatch",
12276            Some(sequence),
12277            Some(recorded_occurrence_id.to_string()),
12278            Some(current_occurrence_id.to_string()),
12279            "recorded condition occurrence differs from the current authored wait position",
12280        )));
12281    }
12282    if recorded_key != Some(current.condition_key.as_str()) {
12283        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12284            "condition_wait_key_mismatch",
12285            Some(sequence),
12286            recorded_key.map(str::to_string),
12287            Some(current.condition_key.clone()),
12288            "recorded condition identity differs from the current workflow wait",
12289        )));
12290    }
12291    if recorded_predicate_identity != current.predicate_identity {
12292        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12293            "condition_wait_predicate_mismatch",
12294            Some(sequence),
12295            Some(recorded_predicate_identity.to_string()),
12296            Some(current.predicate_identity.clone()),
12297            "recorded condition predicate behavior differs from current workflow code",
12298        )));
12299    }
12300    if recorded_timeout_seconds != current.timeout_seconds {
12301        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12302            "condition_wait_timeout_mismatch",
12303            Some(sequence),
12304            recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12305            current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12306            "recorded condition timeout differs from the current workflow wait",
12307        )));
12308    }
12309    Ok(())
12310}
12311
12312/// Future returned by [`WorkflowContext::start_child_workflow`].
12313pub struct ChildWorkflowCall {
12314    ctx: WorkflowContext,
12315    workflow_type: String,
12316    options: ChildWorkflowOptions,
12317    args: Option<Result<AvroValue>>,
12318    scheduled: bool,
12319    matched_pending: bool,
12320    parallel_group_path: Vec<ParallelGroupMetadata>,
12321}
12322
12323impl ChildWorkflowCall {
12324    fn poll_avro_value(
12325        mut self: Pin<&mut Self>,
12326        _cx: &mut TaskContext<'_>,
12327    ) -> Poll<Result<ChildWorkflowAvroResult>> {
12328        if self.matched_pending {
12329            return Poll::Pending;
12330        }
12331
12332        let ctx = self.ctx.clone();
12333        let mut state = match ctx.state.lock() {
12334            Ok(state) => state,
12335            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12336        };
12337
12338        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12339            let sequence = recorded.sequence();
12340            match recorded {
12341                RecordedCommand::ChildWorkflow {
12342                    workflow_type,
12343                    outcome,
12344                    parallel_group_path,
12345                    ..
12346                } => {
12347                    if let Err(error) = ensure_parallel_path_matches(
12348                        sequence,
12349                        parallel_group_path.as_deref(),
12350                        &self.parallel_group_path,
12351                    ) {
12352                        return Poll::Ready(Err(error));
12353                    }
12354                    if let Some(recorded_type) = workflow_type {
12355                        if recorded_type != self.workflow_type {
12356                            return Poll::Ready(Err(Error::NonDeterministicReplay(
12357                                ReplayFailure::new(
12358                                    "recorded_command_detail_mismatch",
12359                                    Some(sequence),
12360                                    Some(format!("child workflow:{recorded_type}")),
12361                                    Some(format!("child workflow:{}", self.workflow_type)),
12362                                    "recorded child workflow type differs from the current workflow command",
12363                                ),
12364                            )));
12365                        }
12366                    }
12367                    state.command_cursor += 1;
12368                    if let Some(outcome) = outcome {
12369                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12370                    }
12371                    state.matched_recorded_pending = true;
12372                    self.scheduled = true;
12373                    self.matched_pending = true;
12374                    return Poll::Pending;
12375                }
12376                other => {
12377                    return Poll::Ready(Err(command_mismatch(
12378                        &other,
12379                        format!("child workflow:{}", self.workflow_type),
12380                    )));
12381                }
12382            }
12383        }
12384
12385        if !self.scheduled {
12386            if self.options.task_queue.trim().is_empty() {
12387                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12388                    "task_queue must not be empty".to_string(),
12389                )));
12390            }
12391            for (name, value) in [
12392                (
12393                    "execution_timeout_seconds",
12394                    self.options.execution_timeout_seconds,
12395                ),
12396                ("run_timeout_seconds", self.options.run_timeout_seconds),
12397            ] {
12398                if value == Some(0) {
12399                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12400                        "{name} must be at least 1"
12401                    ))));
12402                }
12403            }
12404
12405            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12406                Ok(args) => args,
12407                Err(error) => return Poll::Ready(Err(error)),
12408            };
12409            let arguments = match encode_typed_envelope(
12410                &normalize_avro_arguments(args),
12411                &state.payload_codec,
12412            ) {
12413                Ok(arguments) => arguments,
12414                Err(error) => return Poll::Ready(Err(error)),
12415            };
12416            let mut command = json!({
12417                "type": "start_child_workflow",
12418                "workflow_type": self.workflow_type,
12419                "queue": self.options.task_queue,
12420                "parent_close_policy": self.options.parent_close_policy.as_str(),
12421                "arguments": arguments,
12422            });
12423            let object = command
12424                .as_object_mut()
12425                .expect("child workflow command is always an object");
12426            if let Some(policy) = &self.options.retry_policy {
12427                let mut retry_policy = serde_json::Map::new();
12428                if let Some(max_attempts) = policy.max_attempts {
12429                    if max_attempts == 0 {
12430                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12431                            "retry_policy.max_attempts must be at least 1".to_string(),
12432                        )));
12433                    }
12434                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12435                }
12436                if !policy.backoff_seconds.is_empty() {
12437                    retry_policy
12438                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12439                }
12440                if !policy.non_retryable_error_types.is_empty() {
12441                    retry_policy.insert(
12442                        "non_retryable_error_types".to_string(),
12443                        json!(policy.non_retryable_error_types),
12444                    );
12445                }
12446                if retry_policy.is_empty() {
12447                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12448                        "retry_policy must configure at least one field".to_string(),
12449                    )));
12450                }
12451                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12452            }
12453            if let Some(seconds) = self.options.execution_timeout_seconds {
12454                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12455            }
12456            if let Some(seconds) = self.options.run_timeout_seconds {
12457                object.insert("run_timeout_seconds".to_string(), json!(seconds));
12458            }
12459            apply_parallel_group_path(object, &self.parallel_group_path);
12460            state.commands.push(command);
12461            self.scheduled = true;
12462        }
12463
12464        Poll::Pending
12465    }
12466}
12467
12468impl Future for ChildWorkflowCall {
12469    type Output = Result<ChildWorkflowResult>;
12470
12471    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12472        match self.poll_avro_value(cx) {
12473            Poll::Ready(Ok(result)) => match result.result.into_json() {
12474                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12475                    parent: result.parent,
12476                    child: result.child,
12477                    child_workflow_type: result.child_workflow_type,
12478                    result: projected,
12479                })),
12480                Err(error) => Poll::Ready(Err(error)),
12481            },
12482            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12483            Poll::Pending => Poll::Pending,
12484        }
12485    }
12486}
12487
12488fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12489    Error::NonDeterministicReplay(ReplayFailure::new(
12490        "recorded_command_mismatch",
12491        Some(recorded.sequence()),
12492        Some(recorded.shape().to_string()),
12493        Some(actual.into()),
12494        "current workflow command does not match the recorded durable command sequence",
12495    ))
12496}
12497
12498pub struct SignalCall {
12499    ctx: WorkflowContext,
12500    signal_name: String,
12501    runtime_reserved_allowed: bool,
12502    opened_wait: bool,
12503    matched_pending: bool,
12504    parallel_group_path: Vec<ParallelGroupMetadata>,
12505}
12506
12507impl SignalCall {
12508    fn poll_avro_value(
12509        mut self: Pin<&mut Self>,
12510        _cx: &mut TaskContext<'_>,
12511    ) -> Poll<Result<Vec<AvroValue>>> {
12512        if self.matched_pending {
12513            return Poll::Pending;
12514        }
12515        if !self.runtime_reserved_allowed {
12516            if let Err(error) = validate_user_signal_name(&self.signal_name) {
12517                return Poll::Ready(Err(error));
12518            }
12519        }
12520
12521        let ctx = self.ctx.clone();
12522        let mut state = match ctx.state.lock() {
12523            Ok(state) => state,
12524            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12525        };
12526
12527        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12528            match recorded {
12529                RecordedCommand::SignalWait {
12530                    sequence,
12531                    signal_name,
12532                    value,
12533                    parallel_group_path,
12534                } => {
12535                    if let Err(error) = ensure_parallel_path_matches(
12536                        sequence,
12537                        parallel_group_path.as_deref(),
12538                        &self.parallel_group_path,
12539                    ) {
12540                        return Poll::Ready(Err(error));
12541                    }
12542                    if signal_name != self.signal_name {
12543                        return Poll::Ready(Err(Error::NonDeterministicReplay(
12544                            ReplayFailure::new(
12545                                "recorded_command_detail_mismatch",
12546                                Some(sequence),
12547                                Some(format!("signal wait:{signal_name}")),
12548                                Some(format!("signal wait:{}", self.signal_name)),
12549                                "recorded signal name differs from the current workflow command",
12550                            ),
12551                        )));
12552                    }
12553
12554                    state.command_cursor += 1;
12555                    if let Some(value) = value {
12556                        return Poll::Ready(Ok(value));
12557                    }
12558                    if state
12559                        .resume_signal
12560                        .as_ref()
12561                        .is_some_and(|signal| signal.signal_name == self.signal_name)
12562                    {
12563                        let signal = state
12564                            .resume_signal
12565                            .take()
12566                            .expect("matching resume signal is present");
12567                        return Poll::Ready(Ok(signal.arguments));
12568                    }
12569
12570                    state.matched_recorded_pending = true;
12571                    self.opened_wait = true;
12572                    self.matched_pending = true;
12573                    return Poll::Pending;
12574                }
12575                other => {
12576                    return Poll::Ready(Err(command_mismatch(
12577                        &other,
12578                        format!("signal wait:{}", self.signal_name),
12579                    )));
12580                }
12581            }
12582        }
12583
12584        if state
12585            .resume_signal
12586            .as_ref()
12587            .is_some_and(|signal| signal.signal_name == self.signal_name)
12588        {
12589            let signal = state
12590                .resume_signal
12591                .take()
12592                .expect("matching resume signal is present");
12593            return Poll::Ready(Ok(signal.arguments));
12594        }
12595
12596        if !self.opened_wait {
12597            let mut command = serde_json::Map::from_iter([
12598                ("type".to_string(), json!("open_signal_wait")),
12599                ("signal_name".to_string(), json!(self.signal_name)),
12600            ]);
12601            apply_parallel_group_path(&mut command, &self.parallel_group_path);
12602            state.commands.push(Value::Object(command));
12603            self.opened_wait = true;
12604        }
12605
12606        Poll::Pending
12607    }
12608}
12609
12610impl Future for SignalCall {
12611    type Output = Result<Vec<Value>>;
12612
12613    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12614        match self.poll_avro_value(cx) {
12615            Poll::Ready(Ok(values)) => Poll::Ready(
12616                values
12617                    .into_iter()
12618                    .map(AvroValue::into_json)
12619                    .collect::<Result<Vec<_>>>(),
12620            ),
12621            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12622            Poll::Pending => Poll::Pending,
12623        }
12624    }
12625}
12626
12627#[derive(Clone, Debug)]
12628pub struct ActivityContext {
12629    client: Client,
12630    pub task_id: String,
12631    pub activity_attempt_id: String,
12632    pub lease_owner: String,
12633    pub activity_type: String,
12634    pub attempt_number: u64,
12635    pub task_queue: String,
12636    pub worker_id: String,
12637}
12638
12639impl ActivityContext {
12640    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12641        self.client
12642            .heartbeat_activity_task(
12643                &self.task_id,
12644                &self.activity_attempt_id,
12645                &self.lease_owner,
12646                details,
12647            )
12648            .await
12649    }
12650}
12651
12652fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12653    validate_payload_codec(codec)?;
12654    match value {
12655        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12656            value, codec,
12657        )?)),
12658        None => Ok(AvroValue::Array(Vec::new())),
12659    }
12660}
12661
12662fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12663    let Some(signal_name) = task
12664        .signal_name
12665        .as_deref()
12666        .filter(|value| !value.is_empty())
12667    else {
12668        return Ok(None);
12669    };
12670    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12671    let AvroValue::Array(arguments) = decoded else {
12672        unreachable!("normalize_avro_arguments always returns an array");
12673    };
12674
12675    Ok(Some(ResumeSignal {
12676        signal_name: signal_name.to_string(),
12677        arguments,
12678    }))
12679}
12680
12681fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12682    validate_payload_codec(&task.payload_codec)?;
12683    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12684    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12685    for event in &task.history_events {
12686        validate_history_event_payloads(event, &task.payload_codec)?;
12687    }
12688    Ok(())
12689}
12690
12691fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12692    validate_payload_codec(&task.payload_codec)?;
12693    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12694}
12695
12696fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12697    validate_payload_codec(&task.payload_codec)?;
12698    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12699    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12700    for event in &task.history_events {
12701        validate_history_event_payloads(event, &task.payload_codec)?;
12702    }
12703
12704    let Some(export) = task.history_export.as_ref() else {
12705        return Ok(());
12706    };
12707    let export_codec = match export.get("payloads") {
12708        Some(payloads) => declared_payload_codec(payloads, "codec")?,
12709        None => None,
12710    }
12711    .unwrap_or(&task.payload_codec);
12712    validate_payload_codec(export_codec)?;
12713
12714    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12715        for event in events {
12716            let event_type = event
12717                .get("event_type")
12718                .or_else(|| event.get("type"))
12719                .and_then(Value::as_str)
12720                .unwrap_or_default();
12721            if let Some(payload) = event.get("payload") {
12722                validate_history_payloads(event_type, payload, export_codec)?;
12723            }
12724        }
12725    }
12726    for signal in export
12727        .get("signals")
12728        .and_then(Value::as_array)
12729        .into_iter()
12730        .flatten()
12731    {
12732        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12733        validate_payload_codec(codec)?;
12734        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12735    }
12736    for activity in export
12737        .get("activities")
12738        .and_then(Value::as_array)
12739        .into_iter()
12740        .flatten()
12741    {
12742        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12743        validate_payload_codec(codec)?;
12744        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12745        validate_optional_inbound_payload(activity.get("result"), codec)?;
12746    }
12747    Ok(())
12748}
12749
12750fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12751    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12752}
12753
12754fn validate_history_payloads(
12755    event_type: &str,
12756    payload: &Value,
12757    fallback_codec: &str,
12758) -> Result<()> {
12759    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12760    validate_payload_codec(codec)?;
12761    for field in history_payload_fields(event_type) {
12762        validate_optional_inbound_payload(payload.get(*field), codec)?;
12763    }
12764    Ok(())
12765}
12766
12767const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12768
12769fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12770    match event_type {
12771        "ActivityCompleted" => &["result"],
12772        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12773        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12774        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12775        "ChildRunCompleted" => &["result", "output"],
12776        "WorkflowCompleted" => &["output"],
12777        "ServiceCallStarted"
12778        | "ServiceCallCompleted"
12779        | "ServiceCallFailed"
12780        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12781        _ => &[],
12782    }
12783}
12784
12785fn signal_history_payload(payload: &Value) -> Option<&Value> {
12786    SIGNAL_HISTORY_PAYLOAD_FIELDS
12787        .iter()
12788        .find_map(|field| payload.get(*field))
12789}
12790
12791fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12792    match value.get(field) {
12793        None => Ok(None),
12794        Some(Value::String(codec)) => Ok(Some(codec)),
12795        Some(_) => Err(invalid_payload_envelope()),
12796    }
12797}
12798
12799fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12800    validate_payload_codec(codec)?;
12801    if let Some(value) = value.filter(|value| !value.is_null()) {
12802        decode_wire_avro_value(value, codec)?;
12803    }
12804    Ok(())
12805}
12806
12807fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12808    let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12809        invalid_recorded_history(
12810            "parallel_group_metadata_invalid",
12811            sequence,
12812            "non-empty parallel_group_id",
12813            &payload.to_string(),
12814            "parallel-group history is missing its stable identity",
12815        )
12816    })?;
12817    let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12818        invalid_recorded_history(
12819            "parallel_group_metadata_invalid",
12820            sequence,
12821            "activity, child, timer, signal, condition, or mixed group kind",
12822            &payload.to_string(),
12823            "parallel-group history is missing its group kind",
12824        )
12825    })?;
12826    if !matches!(
12827        kind.as_str(),
12828        "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12829    ) {
12830        return Err(invalid_recorded_history(
12831            "parallel_group_metadata_invalid",
12832            sequence,
12833            "activity, child, timer, signal, condition, or mixed group kind",
12834            &kind,
12835            "parallel-group history contains an unsupported group kind",
12836        ));
12837    }
12838    let base_sequence = payload
12839        .get("parallel_group_base_sequence")
12840        .and_then(value_as_u64)
12841        .filter(|value| *value > 0)
12842        .ok_or_else(|| {
12843            invalid_recorded_history(
12844                "parallel_group_metadata_invalid",
12845                sequence,
12846                "positive parallel_group_base_sequence",
12847                &payload.to_string(),
12848                "parallel-group history contains an invalid base sequence",
12849            )
12850        })?;
12851    let size = payload
12852        .get("parallel_group_size")
12853        .and_then(value_as_u64)
12854        .and_then(|value| usize::try_from(value).ok())
12855        .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12856        .ok_or_else(|| {
12857            invalid_recorded_history(
12858                "parallel_group_metadata_invalid",
12859                sequence,
12860                "bounded positive parallel_group_size",
12861                &payload.to_string(),
12862                "parallel-group history contains an invalid group size",
12863            )
12864        })?;
12865    let index = payload
12866        .get("parallel_group_index")
12867        .and_then(value_as_u64)
12868        .and_then(|value| usize::try_from(value).ok())
12869        .filter(|value| *value < size)
12870        .ok_or_else(|| {
12871            invalid_recorded_history(
12872                "parallel_group_metadata_invalid",
12873                sequence,
12874                "parallel_group_index within group bounds",
12875                &payload.to_string(),
12876                "parallel-group history contains an invalid member index",
12877            )
12878        })?;
12879    if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12880        return Err(invalid_recorded_history(
12881            "parallel_group_metadata_invalid",
12882            sequence,
12883            "base sequence plus member index equals workflow sequence",
12884            &payload.to_string(),
12885            "parallel-group path does not preserve durable workflow position",
12886        ));
12887    }
12888    let mode = payload
12889        .get("parallel_group_mode")
12890        .and_then(Value::as_str)
12891        .unwrap_or("all");
12892    if !matches!(mode, "all" | "select") {
12893        return Err(invalid_recorded_history(
12894            "parallel_group_metadata_invalid",
12895            sequence,
12896            "parallel group mode all or select",
12897            mode,
12898            "parallel-group history contains an unsupported group mode",
12899        ));
12900    }
12901    let expected_id = if mode == "select" {
12902        format!("select-calls:{base_sequence}:{size}")
12903    } else {
12904        format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12905    };
12906    if group_id != expected_id {
12907        return Err(invalid_recorded_history(
12908            "parallel_group_metadata_invalid",
12909            sequence,
12910            &expected_id,
12911            &group_id,
12912            "parallel-group history contains an incompatible stable group ID",
12913        ));
12914    }
12915    let selection_member_key = if mode == "select" {
12916        Some(selection_key_from_value(
12917            payload.get("selection_member_key"),
12918            sequence,
12919        )?)
12920    } else {
12921        None
12922    };
12923    let selection_member_index = if mode == "select" {
12924        Some(required_parallel_usize(
12925            payload,
12926            "selection_member_index",
12927            sequence,
12928        )?)
12929    } else {
12930        None
12931    };
12932    let selection_member_base_sequence = if mode == "select" {
12933        Some(
12934            payload
12935                .get("selection_member_base_sequence")
12936                .and_then(value_as_u64)
12937                .filter(|value| *value >= base_sequence)
12938                .ok_or_else(|| {
12939                    invalid_recorded_history(
12940                        "parallel_group_metadata_invalid",
12941                        sequence,
12942                        "selection member base within its group",
12943                        &payload.to_string(),
12944                        "selection history contains an invalid member base sequence",
12945                    )
12946                })?,
12947        )
12948    } else {
12949        None
12950    };
12951    let selection_member_size = if mode == "select" {
12952        let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12953        if member_size == 0 {
12954            return Err(invalid_recorded_history(
12955                "parallel_group_metadata_invalid",
12956                sequence,
12957                "positive selection member size",
12958                &payload.to_string(),
12959                "selection history contains an invalid member size",
12960            ));
12961        }
12962        Some(member_size)
12963    } else {
12964        None
12965    };
12966    let selection_member_kind = if mode == "select" {
12967        let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12968            invalid_recorded_history(
12969                "parallel_group_metadata_invalid",
12970                sequence,
12971                "selection member operation kind",
12972                &payload.to_string(),
12973                "selection history is missing its authored member kind",
12974            )
12975        })?;
12976        if !matches!(
12977            kind.as_str(),
12978            "activity" | "child" | "timer" | "signal" | "condition" | "group"
12979        ) {
12980            return Err(invalid_recorded_history(
12981                "parallel_group_metadata_invalid",
12982                sequence,
12983                "activity, child, timer, signal, condition, or group selection member kind",
12984                &kind,
12985                "selection history contains an unsupported member kind",
12986            ));
12987        }
12988        Some(kind)
12989    } else {
12990        None
12991    };
12992    if let (Some(member_base), Some(member_size)) =
12993        (selection_member_base_sequence, selection_member_size)
12994    {
12995        let member_end = member_base
12996            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12997            .ok_or_else(|| {
12998                invalid_recorded_history(
12999                    "parallel_group_metadata_invalid",
13000                    sequence,
13001                    "bounded selection member range",
13002                    &payload.to_string(),
13003                    "selection member range overflowed",
13004                )
13005            })?;
13006        let group_end = base_sequence
13007            .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
13008            .unwrap_or(u64::MAX);
13009        if sequence < member_base || sequence >= member_end || member_end > group_end {
13010            return Err(invalid_recorded_history(
13011                "parallel_group_metadata_invalid",
13012                sequence,
13013                "workflow sequence within one bounded selection member",
13014                &payload.to_string(),
13015                "selection member range does not contain its durable leaf",
13016            ));
13017        }
13018    }
13019    Ok(ParallelGroupMetadata {
13020        parallel_group_id: group_id,
13021        parallel_group_kind: kind,
13022        parallel_group_base_sequence: base_sequence,
13023        parallel_group_size: size,
13024        parallel_group_index: index,
13025        parallel_group_mode: (mode == "select").then(|| "select".to_string()),
13026        selection_member_key,
13027        selection_member_index,
13028        selection_member_base_sequence,
13029        selection_member_size,
13030        selection_member_kind,
13031    })
13032}
13033
13034fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
13035    payload
13036        .get(field)
13037        .and_then(value_as_u64)
13038        .and_then(|value| usize::try_from(value).ok())
13039        .ok_or_else(|| {
13040            invalid_recorded_history(
13041                "parallel_group_metadata_invalid",
13042                sequence,
13043                &format!("non-negative integer {field}"),
13044                &payload.to_string(),
13045                "selection history contains invalid member metadata",
13046            )
13047        })
13048}
13049
13050fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
13051    match value {
13052        Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
13053        Some(value) => value_as_u64(value)
13054            .and_then(|value| usize::try_from(value).ok())
13055            .map(SelectionKey::Index)
13056            .ok_or_else(|| {
13057                invalid_recorded_history(
13058                    "selection_member_key_invalid",
13059                    sequence,
13060                    "non-empty string or non-negative integer member key",
13061                    &value.to_string(),
13062                    "selection history contains an invalid member key",
13063                )
13064            }),
13065        None => Err(invalid_recorded_history(
13066            "selection_member_key_missing",
13067            sequence,
13068            "selection_member_key",
13069            "<missing>",
13070            "selection history is missing its stable member key",
13071        )),
13072    }
13073}
13074
13075fn recorded_parallel_group_path(
13076    events: &[&HistoryEvent],
13077    sequence: u64,
13078) -> Result<Option<Vec<ParallelGroupMetadata>>> {
13079    let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
13080    for event in events {
13081        let payload = &event.payload;
13082        let has_metadata = payload.get("parallel_group_path").is_some()
13083            || payload.get("parallel_group_id").is_some()
13084            || payload.get("parallel_group_kind").is_some()
13085            || payload.get("parallel_group_base_sequence").is_some()
13086            || payload.get("parallel_group_size").is_some()
13087            || payload.get("parallel_group_index").is_some()
13088            || payload.get("parallel_group_mode").is_some()
13089            || payload.get("selection_member_key").is_some();
13090        if !has_metadata {
13091            continue;
13092        }
13093
13094        let top_level = recorded_parallel_group_entry(payload, sequence)?;
13095        let path = match payload.get("parallel_group_path") {
13096            None => vec![top_level.clone()],
13097            Some(Value::Array(entries)) if !entries.is_empty() => entries
13098                .iter()
13099                .map(|entry| recorded_parallel_group_entry(entry, sequence))
13100                .collect::<Result<Vec<_>>>()?,
13101            Some(value) => {
13102                return Err(invalid_recorded_history(
13103                    "parallel_group_metadata_invalid",
13104                    sequence,
13105                    "non-empty parallel_group_path list",
13106                    &value.to_string(),
13107                    "parallel-group history contains an invalid group path",
13108                ));
13109            }
13110        };
13111        if path.last() != Some(&top_level) {
13112            return Err(invalid_recorded_history(
13113                "parallel_group_metadata_invalid",
13114                sequence,
13115                &serde_json::to_string(&path.last()).unwrap_or_default(),
13116                &serde_json::to_string(&top_level).unwrap_or_default(),
13117                "parallel-group top-level fields do not match the innermost path entry",
13118            ));
13119        }
13120        if recorded.as_ref().is_some_and(|existing| existing != &path) {
13121            return Err(invalid_recorded_history(
13122                "parallel_group_history_conflict",
13123                sequence,
13124                &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
13125                &serde_json::to_string(&path).unwrap_or_default(),
13126                "parallel-group metadata changed between scheduling and resolution history",
13127            ));
13128        }
13129        recorded = Some(path);
13130    }
13131    Ok(recorded)
13132}
13133
13134fn recorded_commands(
13135    events: &[HistoryEvent],
13136    fallback_codec: &str,
13137    parent: WorkflowIdentity,
13138) -> Result<Vec<RecordedCommand>> {
13139    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
13140    let mut last_new_sequence = None;
13141
13142    for event in events {
13143        let is_activity = matches!(
13144            event.event_type.as_str(),
13145            "ActivityScheduled"
13146                | "ActivityStarted"
13147                | "ActivityHeartbeatRecorded"
13148                | "ActivityRetryScheduled"
13149                | "ActivityCompleted"
13150                | "ActivityFailed"
13151                | "ActivityCancelled"
13152                | "ActivityTimedOut"
13153        );
13154        let is_workflow_timer = matches!(
13155            event.event_type.as_str(),
13156            "TimerScheduled" | "TimerCancelled" | "TimerFired"
13157        ) && !is_internal_timer_event(event);
13158        let is_child_workflow = matches!(
13159            event.event_type.as_str(),
13160            "ChildWorkflowScheduled"
13161                | "ChildRunCompleted"
13162                | "ChildRunFailed"
13163                | "ChildRunCancelled"
13164                | "ChildRunTerminated"
13165        );
13166        let is_signal_wait = is_recorded_signal_wait_event(event);
13167        let is_condition_wait = is_recorded_condition_wait_event(event);
13168        let is_search_attributes = event.event_type == "SearchAttributesUpserted";
13169        let is_side_effect = event.event_type == "SideEffectRecorded";
13170        let is_version_marker = event.event_type == "VersionMarkerRecorded";
13171        let is_memo = event.event_type == "MemoUpserted";
13172        if !is_activity
13173            && !is_workflow_timer
13174            && !is_child_workflow
13175            && !is_signal_wait
13176            && !is_condition_wait
13177            && !is_search_attributes
13178            && !is_side_effect
13179            && !is_version_marker
13180            && !is_memo
13181        {
13182            continue;
13183        }
13184
13185        let sequence = durable_event_sequence(event).ok_or_else(|| {
13186            Error::NonDeterministicReplay(ReplayFailure::new(
13187                "durable_command_sequence_missing",
13188                None,
13189                Some("positive workflow sequence".to_string()),
13190                Some(event.event_type.clone()),
13191                "durable command history event has no workflow sequence",
13192            ))
13193        })?;
13194        if sequence == 0 {
13195            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
13196                "durable_command_sequence_invalid",
13197                Some(sequence),
13198                Some("positive workflow sequence".to_string()),
13199                Some(sequence.to_string()),
13200                "durable command history uses an invalid workflow sequence",
13201            )));
13202        }
13203        if !events_by_sequence.contains_key(&sequence) {
13204            if let Some(previous) = last_new_sequence {
13205                if sequence < previous {
13206                    return Err(invalid_recorded_history(
13207                        "durable_command_sequence_mismatch",
13208                        sequence,
13209                        &format!("workflow sequence greater than {previous}"),
13210                        &sequence.to_string(),
13211                        "durable commands are not strictly ordered by their recorded workflow sequence",
13212                    ));
13213                }
13214            }
13215            last_new_sequence = Some(sequence);
13216        }
13217        events_by_sequence.entry(sequence).or_default().push(event);
13218    }
13219
13220    let commands: Vec<RecordedCommand> = events_by_sequence
13221        .into_iter()
13222        .map(|(sequence, sequence_events)| {
13223            let activity_events: Vec<_> = sequence_events
13224                .iter()
13225                .copied()
13226                .filter(|event| event.event_type.starts_with("Activity"))
13227                .collect();
13228            let timer_events: Vec<_> = sequence_events
13229                .iter()
13230                .copied()
13231                .filter(|event| event.event_type.starts_with("Timer"))
13232                .collect();
13233            let child_events: Vec<_> = sequence_events
13234                .iter()
13235                .copied()
13236                .filter(|event| {
13237                    event.event_type == "ChildWorkflowScheduled"
13238                        || event.event_type.starts_with("ChildRun")
13239                })
13240                .collect();
13241            let signal_wait_events: Vec<_> = sequence_events
13242                .iter()
13243                .copied()
13244                .filter(|event| is_recorded_signal_wait_event(event))
13245                .collect();
13246            let condition_wait_events: Vec<_> = sequence_events
13247                .iter()
13248                .copied()
13249                .filter(|event| is_recorded_condition_wait_event(event))
13250                .collect();
13251            let search_attribute_events: Vec<_> = sequence_events
13252                .iter()
13253                .copied()
13254                .filter(|event| event.event_type == "SearchAttributesUpserted")
13255                .collect();
13256            let side_effect_events: Vec<_> = sequence_events
13257                .iter()
13258                .copied()
13259                .filter(|event| event.event_type == "SideEffectRecorded")
13260                .collect();
13261            let version_marker_events: Vec<_> = sequence_events
13262                .iter()
13263                .copied()
13264                .filter(|event| event.event_type == "VersionMarkerRecorded")
13265                .collect();
13266            let memo_events: Vec<_> = sequence_events
13267                .iter()
13268                .copied()
13269                .filter(|event| event.event_type == "MemoUpserted")
13270                .collect();
13271
13272            let command_kind_count = usize::from(!activity_events.is_empty())
13273                + usize::from(!timer_events.is_empty())
13274                + usize::from(!child_events.is_empty())
13275                + usize::from(!signal_wait_events.is_empty())
13276                + usize::from(!condition_wait_events.is_empty())
13277                + usize::from(!search_attribute_events.is_empty())
13278                + usize::from(!side_effect_events.is_empty())
13279                + usize::from(!version_marker_events.is_empty())
13280                + usize::from(!memo_events.is_empty());
13281            if command_kind_count > 1 {
13282                let actual = [
13283                    (!activity_events.is_empty()).then_some("activity"),
13284                    (!timer_events.is_empty()).then_some("timer"),
13285                    (!child_events.is_empty()).then_some("child workflow"),
13286                    (!signal_wait_events.is_empty()).then_some("signal wait"),
13287                    (!condition_wait_events.is_empty()).then_some("condition wait"),
13288                    (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13289                    (!side_effect_events.is_empty()).then_some("side effect"),
13290                    (!version_marker_events.is_empty()).then_some("version marker"),
13291                    (!memo_events.is_empty()).then_some("memo upsert"),
13292                ]
13293                .into_iter()
13294                .flatten()
13295                .collect::<Vec<_>>()
13296                .join(" and ");
13297                return Err(invalid_recorded_history(
13298                    "durable_command_sequence_collision",
13299                    sequence,
13300                    "one durable command kind",
13301                    &actual,
13302                    "one workflow sequence records more than one durable command kind",
13303                ));
13304            }
13305
13306            if !activity_events.is_empty() {
13307                let parallel_group_path =
13308                    recorded_parallel_group_path(&activity_events, sequence)?;
13309                let scheduled_count = activity_events
13310                    .iter()
13311                    .filter(|event| event.event_type == "ActivityScheduled")
13312                    .count();
13313                if scheduled_count > 1 {
13314                    return Err(invalid_recorded_history(
13315                        "duplicate_activity_schedule",
13316                        sequence,
13317                        "at most one ActivityScheduled event",
13318                        "multiple ActivityScheduled events",
13319                        "activity history schedules more than one command at one workflow sequence",
13320                    ));
13321                }
13322                let activity_type = activity_events.iter().find_map(|event| {
13323                    event
13324                        .payload
13325                        .get("activity_type")
13326                        .or_else(|| event.payload.get("activity_name"))
13327                        .and_then(Value::as_str)
13328                        .map(str::to_string)
13329                });
13330                if activity_events.iter().filter_map(|event| {
13331                    event
13332                        .payload
13333                        .get("activity_type")
13334                        .or_else(|| event.payload.get("activity_name"))
13335                        .and_then(Value::as_str)
13336                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13337                    return Err(invalid_recorded_history(
13338                        "activity_identity_mismatch",
13339                        sequence,
13340                        activity_type.as_deref().unwrap_or("one activity identity"),
13341                        "conflicting activity identities",
13342                        "activity lifecycle events at one workflow sequence disagree on identity",
13343                    ));
13344                }
13345                let terminal: Vec<_> = activity_events
13346                    .iter()
13347                    .copied()
13348                    .filter(|event| {
13349                        matches!(
13350                            event.event_type.as_str(),
13351                            "ActivityCompleted"
13352                                | "ActivityFailed"
13353                                | "ActivityCancelled"
13354                                | "ActivityTimedOut"
13355                        )
13356                    })
13357                    .collect();
13358                let duplicate_delivery = terminal.first().is_some_and(|first| {
13359                    terminal.iter().all(|event| {
13360                        event.event_type == first.event_type && event.payload == first.payload
13361                    })
13362                });
13363                if terminal.len() > 1 && !duplicate_delivery {
13364                    return Err(invalid_recorded_history(
13365                        "duplicate_activity_terminal_event",
13366                        sequence,
13367                        "at most one terminal activity event",
13368                        "multiple terminal activity events",
13369                        "activity history settles one command more than once",
13370                    ));
13371                }
13372                let outcome = terminal
13373                    .first()
13374                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13375                    .transpose()?;
13376                let options = activity_events
13377                    .iter()
13378                    .find(|event| event.event_type == "ActivityScheduled")
13379                    .and_then(|event| event.payload.get("activity"))
13380                    .and_then(Value::as_object)
13381                    .map(|activity| RecordedActivityOptions {
13382                        task_queue: recorded_optional_string(activity, "queue"),
13383                        execution_mode: recorded_optional_string(activity, "execution_mode"),
13384                        retry_policy: recorded_activity_retry_snapshot(
13385                            activity.get("retry_policy"),
13386                        ),
13387                    });
13388                return Ok(RecordedCommand::Activity {
13389                    sequence,
13390                    activity_type,
13391                    options,
13392                    outcome,
13393                    parallel_group_path,
13394                });
13395            }
13396
13397            if !child_events.is_empty() {
13398                let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13399                let scheduled: Vec<_> = child_events
13400                    .iter()
13401                    .copied()
13402                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
13403                    .collect();
13404                if scheduled.len() != 1 {
13405                    return Err(invalid_recorded_history(
13406                        "child_workflow_schedule_missing_or_duplicate",
13407                        sequence,
13408                        "one ChildWorkflowScheduled event",
13409                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13410                        "child workflow replay requires exactly one recorded schedule event",
13411                    ));
13412                }
13413                let workflow_type = child_events.iter().find_map(|event| {
13414                    event
13415                        .payload
13416                        .get("child_workflow_type")
13417                        .or_else(|| event.payload.get("workflow_type"))
13418                        .and_then(Value::as_str)
13419                        .filter(|value| !value.is_empty())
13420                        .map(str::to_string)
13421                });
13422                if child_events
13423                    .iter()
13424                    .filter_map(|event| {
13425                        event
13426                            .payload
13427                            .get("child_workflow_type")
13428                            .or_else(|| event.payload.get("workflow_type"))
13429                            .and_then(Value::as_str)
13430                    })
13431                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
13432                {
13433                    return Err(invalid_recorded_history(
13434                        "child_workflow_identity_mismatch",
13435                        sequence,
13436                        workflow_type
13437                            .as_deref()
13438                            .unwrap_or("one child workflow type"),
13439                        "conflicting child workflow types",
13440                        "child workflow lifecycle events at one sequence disagree on type",
13441                    ));
13442                }
13443                let mut outcomes = child_workflow_outcomes(
13444                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13445                    fallback_codec,
13446                    parent.clone(),
13447                )?;
13448                let terminal_events = child_events
13449                    .iter()
13450                    .copied()
13451                    .filter(|event| event.event_type.starts_with("ChildRun"))
13452                    .collect::<Vec<_>>();
13453                let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13454                    terminal_events.iter().all(|event| {
13455                        event.event_type == first.event_type && event.payload == first.payload
13456                    })
13457                });
13458                if outcomes.len() > 1 && !duplicate_delivery {
13459                    return Err(invalid_recorded_history(
13460                        "duplicate_child_workflow_terminal_event",
13461                        sequence,
13462                        "at most one terminal child event",
13463                        "multiple terminal child events",
13464                        "child workflow history settles one command more than once",
13465                    ));
13466                }
13467                return Ok(RecordedCommand::ChildWorkflow {
13468                    sequence,
13469                    workflow_type,
13470                    outcome: outcomes.pop(),
13471                    parallel_group_path,
13472                });
13473            }
13474
13475            if !signal_wait_events.is_empty() {
13476                let opened: Vec<_> = signal_wait_events
13477                    .iter()
13478                    .copied()
13479                    .filter(|event| event.event_type == "SignalWaitOpened")
13480                    .collect();
13481                if opened.len() != 1 {
13482                    return Err(invalid_recorded_history(
13483                        "signal_wait_open_missing_or_duplicate",
13484                        sequence,
13485                        "one SignalWaitOpened event",
13486                        &format!("{} SignalWaitOpened events", opened.len()),
13487                        "signal replay requires exactly one canonical wait-open event",
13488                    ));
13489                }
13490
13491                let applied: Vec<_> = signal_wait_events
13492                    .iter()
13493                    .copied()
13494                    .filter(|event| event.event_type == "SignalApplied")
13495                    .collect();
13496                if applied.len() > 1 {
13497                    return Err(invalid_recorded_history(
13498                        "duplicate_signal_wait_apply",
13499                        sequence,
13500                        "at most one SignalApplied event",
13501                        "multiple SignalApplied events",
13502                        "signal history applies one durable wait more than once",
13503                    ));
13504                }
13505
13506                let signal_names = signal_wait_events
13507                    .iter()
13508                    .map(|event| required_signal_wait_name(event, sequence))
13509                    .collect::<Result<Vec<_>>>()?;
13510                let signal_name = signal_names
13511                    .first()
13512                    .expect("signal wait events are not empty")
13513                    .clone();
13514                if signal_names.iter().any(|candidate| candidate != &signal_name) {
13515                    return Err(invalid_recorded_history(
13516                        "signal_wait_identity_mismatch",
13517                        sequence,
13518                        &signal_name,
13519                        "conflicting signal names",
13520                        "signal wait lifecycle events at one workflow sequence disagree on identity",
13521                    ));
13522                }
13523                let value = applied
13524                    .first()
13525                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
13526                    .transpose()?;
13527                return Ok(RecordedCommand::SignalWait {
13528                    sequence,
13529                    signal_name,
13530                    value,
13531                    parallel_group_path: recorded_parallel_group_path(
13532                        &signal_wait_events,
13533                        sequence,
13534                    )?,
13535                });
13536            }
13537
13538            if !condition_wait_events.is_empty() {
13539                return recorded_condition_wait(
13540                    sequence,
13541                    &condition_wait_events,
13542                    events,
13543                );
13544            }
13545
13546            if !search_attribute_events.is_empty() {
13547                if search_attribute_events.len() != 1 {
13548                    return Err(invalid_recorded_history(
13549                        "duplicate_search_attribute_update",
13550                        sequence,
13551                        "one SearchAttributesUpserted event",
13552                        &format!(
13553                            "{} SearchAttributesUpserted events",
13554                            search_attribute_events.len()
13555                        ),
13556                        "search-attribute history records one workflow command more than once",
13557                    ));
13558                }
13559                let payload = &search_attribute_events[0].payload;
13560                let attributes = payload
13561                    .get("attributes")
13562                    .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13563                    .cloned()
13564                    .ok_or_else(|| {
13565                        invalid_recorded_history(
13566                            "search_attribute_update_missing",
13567                            sequence,
13568                            "non-empty attributes object",
13569                            "missing or invalid attributes",
13570                            "search-attribute history is missing its recorded mutation",
13571                        )
13572                    })?;
13573                let attribute_types =
13574                    recorded_search_attribute_types(payload, &attributes, sequence)?;
13575                return Ok(RecordedCommand::SearchAttributes {
13576                    sequence,
13577                    attributes,
13578                    attribute_types,
13579                });
13580            }
13581
13582            if !side_effect_events.is_empty() {
13583                if side_effect_events.len() != 1 {
13584                    return Err(invalid_recorded_history(
13585                        "duplicate_side_effect_record",
13586                        sequence,
13587                        "one SideEffectRecorded event",
13588                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
13589                        "side-effect history records one workflow command more than once",
13590                    ));
13591                }
13592                let event = side_effect_events[0];
13593                let result = event.payload.get("result").ok_or_else(|| {
13594                    invalid_recorded_history(
13595                        "side_effect_result_missing",
13596                        sequence,
13597                        "recorded result payload",
13598                        "missing result",
13599                        "side-effect history is missing its recorded value",
13600                    )
13601                })?;
13602                let has_published_envelope = result.as_str().is_some()
13603                    || result.as_object().is_some_and(|envelope| {
13604                        envelope.get("codec").and_then(Value::as_str).is_some()
13605                            && envelope.get("blob").and_then(Value::as_str).is_some()
13606                    });
13607                if !has_published_envelope {
13608                    return Err(invalid_recorded_history(
13609                        "side_effect_payload_malformed",
13610                        sequence,
13611                        "payload blob or {codec, blob} envelope",
13612                        &result.to_string(),
13613                        "side-effect history result does not use a published payload envelope",
13614                    ));
13615                }
13616                let codec = event
13617                    .payload
13618                    .get("payload_codec")
13619                    .and_then(Value::as_str)
13620                    .unwrap_or(fallback_codec);
13621                let value = decode_wire_avro_value(result, codec).map_err(|error| {
13622                    if error.to_string().contains("unsupported_payload_codec") {
13623                        return error;
13624                    }
13625
13626                    invalid_recorded_history(
13627                        "side_effect_payload_incompatible",
13628                        sequence,
13629                        &format!("valid {codec} payload envelope"),
13630                        &error.to_string(),
13631                        "side-effect history payload cannot be decoded with its recorded codec",
13632                    )
13633                })?;
13634                return Ok(RecordedCommand::SideEffect { sequence, value });
13635            }
13636
13637            if !version_marker_events.is_empty() {
13638                if version_marker_events.len() != 1 {
13639                    return Err(invalid_recorded_history(
13640                        "duplicate_version_marker_record",
13641                        sequence,
13642                        "one VersionMarkerRecorded event",
13643                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13644                        "version-marker history records one workflow command more than once",
13645                    ));
13646                }
13647                let payload = &version_marker_events[0].payload;
13648                let change_id = payload
13649                    .get("change_id")
13650                    .and_then(Value::as_str)
13651                    .filter(|value| !value.is_empty())
13652                    .map(str::to_string)
13653                    .ok_or_else(|| {
13654                        invalid_recorded_history(
13655                            "version_marker_field_missing",
13656                            sequence,
13657                            "non-empty change_id",
13658                            "missing or invalid change_id",
13659                            "version-marker history is missing its stable change ID",
13660                        )
13661                    })?;
13662                let version = required_version_i32(payload, "version", sequence)?;
13663                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13664                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13665                if min_supported > max_supported || version < min_supported || version > max_supported {
13666                    return Err(invalid_recorded_history(
13667                        "version_marker_history_range_invalid",
13668                        sequence,
13669                        "min_supported <= version <= max_supported",
13670                        &format!("{min_supported} <= {version} <= {max_supported}"),
13671                        "recorded version marker contains an internally incompatible range",
13672                    ));
13673                }
13674                return Ok(RecordedCommand::VersionMarker {
13675                    sequence,
13676                    change_id,
13677                    version,
13678                });
13679            }
13680
13681            if !memo_events.is_empty() {
13682                if memo_events.len() != 1 {
13683                    return Err(invalid_recorded_history(
13684                        "duplicate_memo_upsert_record",
13685                        sequence,
13686                        "one MemoUpserted event",
13687                        &format!("{} MemoUpserted events", memo_events.len()),
13688                        "memo history records one workflow update more than once",
13689                    ));
13690                }
13691                let payload = &memo_events[0].payload;
13692                let entries = payload.get("entries").cloned().ok_or_else(|| {
13693                    invalid_recorded_history(
13694                        "memo_entries_missing",
13695                        sequence,
13696                        "memo entries object",
13697                        "missing entries",
13698                        "MemoUpserted history is missing replay identity entries",
13699                    )
13700                })?;
13701                let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13702                    invalid_recorded_history(
13703                        "memo_entries_invalid",
13704                        sequence,
13705                        "valid canonical memo entries",
13706                        &error.to_string(),
13707                        "MemoUpserted history contains invalid replay identity entries",
13708                    )
13709                })?;
13710                let merged = payload.get("merged").cloned().ok_or_else(|| {
13711                    invalid_recorded_history(
13712                        "memo_merged_projection_missing",
13713                        sequence,
13714                        "merged memo projection",
13715                        "missing merged",
13716                        "MemoUpserted history is missing its merged projection",
13717                    )
13718                })?;
13719                decode_memo_history_map(&merged, false).map_err(|error| {
13720                    invalid_recorded_history(
13721                        "memo_merged_projection_invalid",
13722                        sequence,
13723                        "valid merged memo projection",
13724                        &error.to_string(),
13725                        "MemoUpserted history contains an invalid merged projection",
13726                    )
13727                })?;
13728
13729                return Ok(RecordedCommand::Memo { sequence, entries });
13730            }
13731            let scheduled: Vec<_> = timer_events
13732                .iter()
13733                .copied()
13734                .filter(|event| event.event_type == "TimerScheduled")
13735                .collect();
13736            let fired: Vec<_> = timer_events
13737                .iter()
13738                .copied()
13739                .filter(|event| event.event_type == "TimerFired")
13740                .collect();
13741            if scheduled.len() != 1 {
13742                return Err(invalid_recorded_history(
13743                    "timer_schedule_missing_or_duplicate",
13744                    sequence,
13745                    "one TimerScheduled event",
13746                    &format!("{} TimerScheduled events", scheduled.len()),
13747                    "timer replay requires exactly one recorded schedule event",
13748                ));
13749            }
13750            if fired.len() > 1 {
13751                return Err(invalid_recorded_history(
13752                    "duplicate_timer_fire",
13753                    sequence,
13754                    "at most one TimerFired event",
13755                    "multiple TimerFired events",
13756                    "timer history contains more than one fire event for a workflow sequence",
13757                ));
13758            }
13759
13760            let scheduled = scheduled[0];
13761            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13762            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13763            if let Some(fired) = fired.first() {
13764                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13765                if fired_timer_id != timer_id {
13766                    return Err(invalid_recorded_history(
13767                        "timer_identity_mismatch",
13768                        sequence,
13769                        &timer_id,
13770                        &fired_timer_id,
13771                        "TimerFired does not correspond to the recorded TimerScheduled event",
13772                    ));
13773                }
13774                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13775                if fired_delay != delay_seconds {
13776                    return Err(invalid_recorded_history(
13777                        "timer_history_delay_mismatch",
13778                        sequence,
13779                        &delay_seconds.to_string(),
13780                        &fired_delay.to_string(),
13781                        "TimerScheduled and TimerFired record different delays",
13782                    ));
13783                }
13784            }
13785
13786            Ok(RecordedCommand::Timer {
13787                sequence,
13788                delay_seconds,
13789                fired: !fired.is_empty(),
13790                parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13791            })
13792        })
13793        .collect::<Result<_>>()?;
13794
13795    let mut marker_sequences = HashMap::new();
13796    for command in &commands {
13797        if let RecordedCommand::VersionMarker {
13798            sequence,
13799            change_id,
13800            ..
13801        } = command
13802        {
13803            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13804                return Err(invalid_recorded_history(
13805                    "duplicate_version_marker",
13806                    *sequence,
13807                    &format!("one marker for change ID {change_id:?}"),
13808                    &format!("markers at sequences {first_sequence} and {sequence}"),
13809                    "workflow history contains duplicate markers for one stable change ID",
13810                ));
13811            }
13812        }
13813    }
13814
13815    Ok(commands)
13816}
13817
13818fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13819    payload
13820        .get(field)
13821        .and_then(Value::as_i64)
13822        .and_then(|value| i32::try_from(value).ok())
13823        .ok_or_else(|| {
13824            invalid_recorded_history(
13825                "version_marker_field_missing",
13826                sequence,
13827                &format!("integer {field}"),
13828                "missing or out-of-range integer",
13829                "version-marker history is missing a required integer field",
13830            )
13831        })
13832}
13833
13834fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13835    event
13836        .payload
13837        .get("sequence")
13838        .or_else(|| event.payload.get("workflow_sequence"))
13839        .or_else(|| event.raw.get("sequence"))
13840        .or_else(|| event.raw.get("workflow_sequence"))
13841        .and_then(value_as_u64)
13842}
13843
13844fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13845    matches!(
13846        event
13847            .payload
13848            .get("timer_kind")
13849            .or_else(|| event.raw.get("timer_kind"))
13850            .and_then(Value::as_str),
13851        Some("condition_timeout" | "signal_timeout")
13852    )
13853}
13854
13855fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13856    matches!(
13857        event.event_type.as_str(),
13858        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13859    )
13860}
13861
13862fn recorded_condition_wait(
13863    sequence: u64,
13864    condition_events: &[&HistoryEvent],
13865    all_events: &[HistoryEvent],
13866) -> Result<RecordedCommand> {
13867    let opened = condition_events
13868        .iter()
13869        .copied()
13870        .filter(|event| event.event_type == "ConditionWaitOpened")
13871        .collect::<Vec<_>>();
13872    if opened.len() != 1 {
13873        return Err(invalid_recorded_history(
13874            "condition_wait_open_missing_or_duplicate",
13875            sequence,
13876            "one ConditionWaitOpened event",
13877            &format!("{} ConditionWaitOpened events", opened.len()),
13878            "condition replay requires exactly one canonical wait-open event",
13879        ));
13880    }
13881    let terminal = condition_events
13882        .iter()
13883        .copied()
13884        .filter(|event| {
13885            matches!(
13886                event.event_type.as_str(),
13887                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13888            )
13889        })
13890        .collect::<Vec<_>>();
13891    if terminal.len() > 1 {
13892        return Err(invalid_recorded_history(
13893            "duplicate_condition_wait_terminal_event",
13894            sequence,
13895            "at most one condition terminal event",
13896            "multiple condition terminal events",
13897            "condition history settles one durable wait more than once",
13898        ));
13899    }
13900
13901    let opened = opened[0];
13902    let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13903    let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13904    for event in condition_events
13905        .iter()
13906        .copied()
13907        .filter(|event| !std::ptr::eq(*event, opened))
13908    {
13909        let event_wait_id = required_condition_wait_id(event, sequence)?;
13910        if event_wait_id != condition_wait_id {
13911            return Err(invalid_recorded_history(
13912                "condition_wait_id_mismatch",
13913                sequence,
13914                &condition_wait_id,
13915                &event_wait_id,
13916                "condition lifecycle events at one sequence disagree on wait identity",
13917            ));
13918        }
13919        let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13920        if event_occurrence_id != occurrence_id {
13921            return Err(invalid_recorded_history(
13922                "condition_wait_occurrence_history_mismatch",
13923                sequence,
13924                &occurrence_id,
13925                &event_occurrence_id,
13926                "condition lifecycle events at one sequence disagree on authored occurrence identity",
13927            ));
13928        }
13929    }
13930
13931    let condition_key = optional_non_empty_history_string(opened, "condition_key");
13932    let predicate_identity = opened
13933        .payload
13934        .get("condition_definition_fingerprint")
13935        .and_then(Value::as_str)
13936        .filter(|value| !value.is_empty())
13937        .map(str::to_string)
13938        .ok_or_else(|| {
13939            invalid_recorded_history(
13940                "condition_wait_predicate_fingerprint_missing",
13941                sequence,
13942                "non-empty condition_definition_fingerprint",
13943                &opened.event_type,
13944                "canonical condition history is missing its predicate identity",
13945            )
13946        })?;
13947    let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13948    for event in condition_events
13949        .iter()
13950        .copied()
13951        .filter(|event| !std::ptr::eq(*event, opened))
13952    {
13953        for (field, opened_value) in [
13954            ("condition_key", condition_key.as_deref()),
13955            (
13956                "condition_definition_fingerprint",
13957                Some(predicate_identity.as_str()),
13958            ),
13959        ] {
13960            if let Some(value) = optional_non_empty_history_string(event, field) {
13961                if opened_value.is_some_and(|opened_value| opened_value != value) {
13962                    return Err(invalid_recorded_history(
13963                        "condition_wait_definition_history_mismatch",
13964                        sequence,
13965                        opened_value.unwrap_or_default(),
13966                        &value,
13967                        "condition lifecycle events disagree on the recorded definition",
13968                    ));
13969                }
13970            }
13971        }
13972        if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13973            if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13974                return Err(invalid_recorded_history(
13975                    "condition_wait_definition_history_mismatch",
13976                    sequence,
13977                    &format!("{}s", timeout_seconds.unwrap_or_default()),
13978                    &format!("{event_timeout}s"),
13979                    "condition lifecycle events disagree on the recorded timeout",
13980                ));
13981            }
13982        }
13983    }
13984
13985    let timeout_timer_events = all_events
13986        .iter()
13987        .filter(|event| {
13988            matches!(
13989                event.event_type.as_str(),
13990                "TimerScheduled" | "TimerCancelled" | "TimerFired"
13991            ) && event.payload.get("timer_kind").and_then(Value::as_str)
13992                == Some("condition_timeout")
13993                && event
13994                    .payload
13995                    .get("condition_wait_id")
13996                    .and_then(Value::as_str)
13997                    == Some(condition_wait_id.as_str())
13998        })
13999        .collect::<Vec<_>>();
14000    let scheduled = timeout_timer_events
14001        .iter()
14002        .copied()
14003        .filter(|event| event.event_type == "TimerScheduled")
14004        .collect::<Vec<_>>();
14005    let fired = timeout_timer_events
14006        .iter()
14007        .copied()
14008        .filter(|event| event.event_type == "TimerFired")
14009        .collect::<Vec<_>>();
14010    if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
14011        return Err(invalid_recorded_history(
14012            "condition_wait_timeout_history_invalid",
14013            sequence,
14014            "one timeout schedule and at most one fire",
14015            &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
14016            "condition timeout history has a missing or duplicate lifecycle event",
14017        ));
14018    }
14019    if let Some(scheduled) = scheduled.first() {
14020        let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
14021        let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
14022        if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
14023            return Err(invalid_recorded_history(
14024                "condition_wait_timeout_delay_mismatch",
14025                sequence,
14026                &format!("{}s", timeout_seconds.unwrap_or_default()),
14027                &format!("{delay_seconds}s"),
14028                "condition timeout timer differs from the wait definition",
14029            ));
14030        }
14031        if let Some(fired) = fired.first() {
14032            let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
14033            let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
14034            if fired_timer_id != timer_id || fired_delay != delay_seconds {
14035                return Err(invalid_recorded_history(
14036                    "condition_wait_timeout_identity_mismatch",
14037                    sequence,
14038                    &format!("{timer_id}:{delay_seconds}s"),
14039                    &format!("{fired_timer_id}:{fired_delay}s"),
14040                    "condition timeout fire does not match its durable schedule",
14041                ));
14042            }
14043        }
14044    }
14045
14046    let result = terminal.first().map(|event| {
14047        if event.event_type == "ConditionWaitTimedOut" {
14048            ConditionWaitResult::TimedOut
14049        } else {
14050            ConditionWaitResult::Satisfied
14051        }
14052    });
14053    let result = if !fired.is_empty() {
14054        if result == Some(ConditionWaitResult::Satisfied) {
14055            return Err(invalid_recorded_history(
14056                "condition_wait_terminal_conflict",
14057                sequence,
14058                "one satisfied or timed-out outcome",
14059                "satisfied event and fired timeout",
14060                "condition history records conflicting terminal outcomes",
14061            ));
14062        }
14063        Some(ConditionWaitResult::TimedOut)
14064    } else {
14065        result
14066    };
14067
14068    Ok(RecordedCommand::ConditionWait {
14069        sequence,
14070        occurrence_id,
14071        condition_key,
14072        predicate_identity,
14073        timeout_seconds,
14074        result,
14075        parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
14076    })
14077}
14078
14079fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
14080    event
14081        .payload
14082        .get("condition_wait_occurrence_id")
14083        .and_then(Value::as_str)
14084        .filter(|value| !value.is_empty())
14085        .map(str::to_string)
14086        .ok_or_else(|| {
14087            invalid_recorded_history(
14088                "condition_wait_occurrence_id_missing",
14089                sequence,
14090                "non-empty condition_wait_occurrence_id",
14091                &event.event_type,
14092                "condition history is missing authored occurrence identity",
14093            )
14094        })
14095}
14096
14097fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
14098    event
14099        .payload
14100        .get("condition_wait_id")
14101        .and_then(Value::as_str)
14102        .filter(|value| !value.is_empty())
14103        .map(str::to_string)
14104        .ok_or_else(|| {
14105            invalid_recorded_history(
14106                "condition_wait_id_missing",
14107                sequence,
14108                "non-empty condition_wait_id",
14109                &event.event_type,
14110                "canonical condition history is missing its durable wait identity",
14111            )
14112        })
14113}
14114
14115fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
14116    event
14117        .payload
14118        .get(field)
14119        .and_then(Value::as_str)
14120        .filter(|value| !value.is_empty())
14121        .map(str::to_string)
14122}
14123
14124fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
14125    match event.payload.get(field) {
14126        None | Some(Value::Null) => Ok(None),
14127        Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
14128            invalid_recorded_history(
14129                "condition_wait_definition_invalid",
14130                sequence,
14131                &format!("non-negative integer {field}"),
14132                &value.to_string(),
14133                "condition history contains an invalid numeric definition field",
14134            )
14135        }),
14136    }
14137}
14138
14139fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
14140    event
14141        .payload
14142        .get("signal_name")
14143        .or_else(|| event.raw.get("signal_name"))
14144        .and_then(Value::as_str)
14145        .filter(|value| !value.is_empty())
14146        .map(str::to_string)
14147        .ok_or_else(|| {
14148            invalid_recorded_history(
14149                "signal_wait_name_missing",
14150                sequence,
14151                "non-empty signal_name",
14152                &event.event_type,
14153                "canonical signal-wait history is missing its signal identity",
14154            )
14155        })
14156}
14157
14158fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
14159    matches!(
14160        event.event_type.as_str(),
14161        "SignalWaitOpened" | "SignalApplied"
14162    )
14163}
14164
14165fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
14166    event
14167        .payload
14168        .get(field)
14169        .and_then(Value::as_str)
14170        .filter(|value| !value.is_empty())
14171        .map(str::to_string)
14172        .ok_or_else(|| {
14173            invalid_recorded_history(
14174                "timer_history_field_missing",
14175                sequence,
14176                field,
14177                &event.event_type,
14178                "timer history is missing a required identity field",
14179            )
14180        })
14181}
14182
14183fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
14184    event
14185        .payload
14186        .get(field)
14187        .and_then(value_as_u64)
14188        .ok_or_else(|| {
14189            invalid_recorded_history(
14190                "timer_history_field_missing",
14191                sequence,
14192                field,
14193                &event.event_type,
14194                "timer history is missing a required numeric field",
14195            )
14196        })
14197}
14198
14199fn recorded_search_attribute_types(
14200    payload: &Value,
14201    attributes: &Value,
14202    sequence: u64,
14203) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
14204    let Some(raw_types) = payload.get("attribute_types") else {
14205        // This is the explicit compatibility rule for histories recorded
14206        // before typed identity was persisted. Values still constrain replay;
14207        // the unknown type snapshot does not assert a typed match.
14208        return Ok(RecordedSnapshotValue::Unknown);
14209    };
14210    let Some(raw_types) = raw_types.as_object() else {
14211        return Err(invalid_recorded_history(
14212            "search_attribute_types_malformed",
14213            sequence,
14214            "canonical attribute type map",
14215            &raw_types.to_string(),
14216            "search-attribute history contains malformed type identity",
14217        ));
14218    };
14219    let attribute_keys = attributes
14220        .as_object()
14221        .expect("recorded search attributes were validated as an object");
14222    let mut types = BTreeMap::new();
14223    for (key, value) in raw_types {
14224        let Some(attribute_type) = value.as_str() else {
14225            return Err(invalid_recorded_history(
14226                "search_attribute_types_malformed",
14227                sequence,
14228                "canonical string type name",
14229                &value.to_string(),
14230                "search-attribute history contains a non-string type identity",
14231            ));
14232        };
14233        if !attribute_keys.contains_key(key)
14234            || !matches!(
14235                attribute_type,
14236                "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
14237            )
14238        {
14239            return Err(invalid_recorded_history(
14240                "search_attribute_types_malformed",
14241                sequence,
14242                "canonical types for keys present in attributes",
14243                &format!("{key}:{attribute_type}"),
14244                "search-attribute history contains unsupported or orphaned type identity",
14245            ));
14246        }
14247        types.insert(key.clone(), attribute_type.to_string());
14248    }
14249    Ok(RecordedSnapshotValue::Known(types))
14250}
14251
14252fn invalid_recorded_history(
14253    reason: &str,
14254    sequence: u64,
14255    expected: &str,
14256    actual: &str,
14257    message: &str,
14258) -> Error {
14259    Error::NonDeterministicReplay(ReplayFailure::new(
14260        reason,
14261        Some(sequence),
14262        Some(expected.to_string()),
14263        Some(actual.to_string()),
14264        message,
14265    ))
14266}
14267
14268type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
14269
14270fn activity_outcome(
14271    event: &HistoryEvent,
14272    fallback_codec: &str,
14273    recorded_activity_type: Option<String>,
14274) -> Result<ActivityOutcome> {
14275    if event.event_type == "ActivityCompleted" {
14276        let codec = event
14277            .payload
14278            .get("payload_codec")
14279            .and_then(Value::as_str)
14280            .unwrap_or(fallback_codec);
14281        return Ok(Ok(decode_wire_avro_value(
14282            event.payload.get("result").unwrap_or(&Value::Null),
14283            codec,
14284        )?));
14285    }
14286
14287    let payload = &event.payload;
14288    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14289        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14290        "ActivityCancelled" => (
14291            ActivityFailureKind::Cancelled,
14292            "cancelled",
14293            "activity was cancelled",
14294        ),
14295        "ActivityTimedOut" => (
14296            ActivityFailureKind::TimedOut,
14297            "timeout",
14298            "activity timed out",
14299        ),
14300        _ => unreachable!("activity_outcome is called only for terminal activity events"),
14301    };
14302    let exception = payload
14303        .get("exception")
14304        .filter(|value| !value.is_null())
14305        .cloned();
14306    let failure_category = payload_string(payload, "failure_category");
14307    let timeout_kind = payload_string(payload, "timeout_kind");
14308    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14309        ActivityFailureKind::Failed => failure_category
14310            .clone()
14311            .unwrap_or_else(|| fallback_reason.to_string()),
14312        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14313        ActivityFailureKind::TimedOut => timeout_kind
14314            .clone()
14315            .unwrap_or_else(|| fallback_reason.to_string()),
14316    });
14317    let message = payload_string(payload, "message")
14318        .or_else(|| {
14319            exception
14320                .as_ref()
14321                .and_then(|value| payload_string(value, "message"))
14322        })
14323        .unwrap_or_else(|| fallback_message.to_string());
14324
14325    Ok(Err(ActivityFailure {
14326        kind,
14327        reason,
14328        message,
14329        activity_execution_id: payload_string(payload, "activity_execution_id"),
14330        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14331        activity_type: payload_string(payload, "activity_type")
14332            .or_else(|| payload_string(payload, "activity_name"))
14333            .or(recorded_activity_type),
14334        activity_class: payload_string(payload, "activity_class"),
14335        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14336        failure_id: payload_string(payload, "failure_id"),
14337        failure_category,
14338        timeout_kind,
14339        non_retryable: payload
14340            .get("non_retryable")
14341            .and_then(Value::as_bool)
14342            .unwrap_or(false),
14343        exception_type: payload_string(payload, "exception_type").or_else(|| {
14344            exception
14345                .as_ref()
14346                .and_then(|value| payload_string(value, "type"))
14347        }),
14348        exception_class: payload_string(payload, "exception_class").or_else(|| {
14349            exception
14350                .as_ref()
14351                .and_then(|value| payload_string(value, "class"))
14352        }),
14353        code: payload
14354            .get("code")
14355            .filter(|value| !value.is_null())
14356            .cloned(),
14357        exception,
14358    }))
14359}
14360
14361type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14362
14363fn child_workflow_outcomes(
14364    events: &[HistoryEvent],
14365    fallback_codec: &str,
14366    parent: WorkflowIdentity,
14367) -> Result<Vec<ChildWorkflowOutcome>> {
14368    let mut outcomes = Vec::new();
14369
14370    for event in events {
14371        let kind = match event.event_type.as_str() {
14372            "ChildRunCompleted" => None,
14373            "ChildRunFailed" => Some((
14374                ChildWorkflowFailureKind::Failed,
14375                "child_workflow",
14376                "child workflow failed",
14377            )),
14378            "ChildRunCancelled" => Some((
14379                ChildWorkflowFailureKind::Cancelled,
14380                "cancelled",
14381                "child workflow was cancelled",
14382            )),
14383            "ChildRunTerminated" => Some((
14384                ChildWorkflowFailureKind::Terminated,
14385                "terminated",
14386                "child workflow was terminated",
14387            )),
14388            _ => continue,
14389        };
14390        let payload = &event.payload;
14391        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14392        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14393        let child_workflow_type = payload_string(payload, "child_workflow_type");
14394
14395        if let Some((kind, reason, fallback_message)) = kind {
14396            let exception = payload
14397                .get("exception")
14398                .filter(|value| !value.is_null())
14399                .cloned();
14400            let message = payload_string(payload, "message")
14401                .or_else(|| {
14402                    exception
14403                        .as_ref()
14404                        .and_then(|value| payload_string(value, "message"))
14405                })
14406                .unwrap_or_else(|| fallback_message.to_string());
14407            let exception_type = payload_string(payload, "exception_type").or_else(|| {
14408                exception
14409                    .as_ref()
14410                    .and_then(|value| payload_string(value, "type"))
14411            });
14412            let exception_class = payload_string(payload, "exception_class").or_else(|| {
14413                exception
14414                    .as_ref()
14415                    .and_then(|value| payload_string(value, "class"))
14416            });
14417            outcomes.push(Err(ChildWorkflowFailure {
14418                kind,
14419                reason: reason.to_string(),
14420                message,
14421                parent_workflow_id: parent.workflow_id.clone(),
14422                parent_workflow_run_id: parent.run_id.clone(),
14423                child_workflow_id,
14424                child_workflow_run_id,
14425                child_workflow_type,
14426                failure_id: payload_string(payload, "failure_id"),
14427                failure_category: payload_string(payload, "failure_category"),
14428                exception_type,
14429                exception_class,
14430                non_retryable: payload
14431                    .get("non_retryable")
14432                    .and_then(Value::as_bool)
14433                    .unwrap_or(false),
14434                code: payload
14435                    .get("code")
14436                    .filter(|value| !value.is_null())
14437                    .cloned(),
14438                exception,
14439            }));
14440            continue;
14441        }
14442
14443        let codec = payload
14444            .get("payload_codec")
14445            .and_then(Value::as_str)
14446            .unwrap_or(fallback_codec);
14447        let result = payload
14448            .get("result")
14449            .or_else(|| payload.get("output"))
14450            .unwrap_or(&Value::Null);
14451        outcomes.push(Ok(ChildWorkflowAvroResult {
14452            parent: parent.clone(),
14453            child: WorkflowIdentity {
14454                workflow_id: child_workflow_id,
14455                run_id: child_workflow_run_id,
14456            },
14457            child_workflow_type,
14458            result: decode_wire_avro_value(result, codec)?,
14459        }));
14460    }
14461
14462    Ok(outcomes)
14463}
14464
14465fn payload_string(payload: &Value, key: &str) -> Option<String> {
14466    payload
14467        .get(key)
14468        .and_then(Value::as_str)
14469        .filter(|value| !value.is_empty())
14470        .map(str::to_string)
14471}
14472
14473fn recorded_activity_failure_boundary(
14474    ctx: &WorkflowContext,
14475    error: &Error,
14476) -> Option<(u64, String)> {
14477    let Error::ActivityFailed(failure) = error else {
14478        return None;
14479    };
14480    if failure.kind != ActivityFailureKind::Failed {
14481        return None;
14482    }
14483    let activity_execution_id = failure
14484        .activity_execution_id
14485        .as_deref()
14486        .filter(|value| !value.is_empty())?;
14487    let state = ctx.state.lock().ok()?;
14488    let mut matches = state.recorded_commands.iter().filter_map(|command| {
14489        let RecordedCommand::Activity {
14490            sequence,
14491            outcome: Some(Err(recorded_failure)),
14492            ..
14493        } = command
14494        else {
14495            return None;
14496        };
14497        (*sequence > 0 && recorded_failure == failure)
14498            .then_some((*sequence, activity_execution_id.to_string()))
14499    });
14500    let boundary = matches.next()?;
14501    matches.next().is_none().then_some(boundary)
14502}
14503
14504fn workflow_failure_command(
14505    error: &Error,
14506    failed_activity_boundary: Option<(u64, String)>,
14507) -> Value {
14508    let (exception_type, exception_class, properties) = match error {
14509        Error::ActivityFailed(failure) => (
14510            match failure.kind {
14511                ActivityFailureKind::Failed => "ActivityFailed",
14512                ActivityFailureKind::Cancelled => "ActivityCancelled",
14513                ActivityFailureKind::TimedOut => "ActivityTimedOut",
14514            },
14515            "durable_workflow::ActivityFailure",
14516            json!({
14517                "reason": failure.reason,
14518                "activity_execution_id": failure.activity_execution_id,
14519                "activity_attempt_id": failure.activity_attempt_id,
14520                "activity_type": failure.activity_type,
14521                "activity_class": failure.activity_class,
14522                "attempt_number": failure.attempt_number,
14523                "failure_id": failure.failure_id,
14524                "failure_category": failure.failure_category,
14525                "timeout_kind": failure.timeout_kind,
14526                "activity_non_retryable": failure.non_retryable,
14527                "activity_exception_type": failure.exception_type,
14528                "activity_exception_class": failure.exception_class,
14529                "activity_code": failure.code,
14530                "activity_exception": failure.exception,
14531            }),
14532        ),
14533        Error::ChildWorkflowFailed(failure) => (
14534            match failure.kind {
14535                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14536                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14537                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14538            },
14539            "durable_workflow::ChildWorkflowFailure",
14540            json!({
14541                "reason": failure.reason,
14542                "parent_workflow_id": failure.parent_workflow_id,
14543                "parent_workflow_run_id": failure.parent_workflow_run_id,
14544                "child_workflow_id": failure.child_workflow_id,
14545                "child_workflow_run_id": failure.child_workflow_run_id,
14546                "child_workflow_type": failure.child_workflow_type,
14547                "failure_id": failure.failure_id,
14548                "failure_category": failure.failure_category,
14549                "child_exception_type": failure.exception_type,
14550                "child_exception_class": failure.exception_class,
14551                "child_non_retryable": failure.non_retryable,
14552                "child_code": failure.code,
14553                "child_exception": failure.exception,
14554            }),
14555        ),
14556        Error::ParallelFailed(failure) => (
14557            "ParallelFailed",
14558            "durable_workflow::ParallelFailure",
14559            json!({
14560                "parallel_group_id": failure.group_id,
14561                "parallel_member_path": failure.member_path,
14562                "parallel_group_path": failure.group_path,
14563                "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14564                "cause_type": workflow_error_type(&failure.cause),
14565                "cause_message": failure.cause.to_string(),
14566            }),
14567        ),
14568        Error::SagaCompensationFailed(failure) => (
14569            "SagaCompensationFailed",
14570            "durable_workflow::SagaCompensationFailure",
14571            json!({
14572                "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14573                "initiating_failure_message": failure.initiating_failure.to_string(),
14574                "compensation_activity_type": failure.compensation_activity_type,
14575                "compensation_registration_order": failure.compensation_registration_order,
14576                "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14577                "compensation_failure_message": failure.compensation_failure.to_string(),
14578            }),
14579        ),
14580        Error::WorkflowCancellationRequested(_) => (
14581            "WorkflowCancellationRequested",
14582            "durable_workflow::WorkflowCancellationRequested",
14583            json!({"reason": "cancelled"}),
14584        ),
14585        Error::NonDeterministicReplay(_) => (
14586            "NonDeterministicReplay",
14587            "durable_workflow::Error",
14588            Value::Null,
14589        ),
14590        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14591    };
14592    let non_retryable = match error {
14593        Error::ActivityFailed(failure) => failure.non_retryable,
14594        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14595        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14596        Error::SagaCompensationFailed(failure) => {
14597            workflow_error_non_retryable(&failure.compensation_failure)
14598        }
14599        Error::WorkflowCancellationRequested(_) => true,
14600        Error::NonDeterministicReplay(_) => true,
14601        _ => false,
14602    };
14603
14604    let mut command = json!({
14605        "type": "fail_workflow",
14606        "message": error.to_string(),
14607        "exception_type": exception_type,
14608        "exception_class": exception_class,
14609        "non_retryable": non_retryable,
14610        "exception": {
14611            "type": exception_type,
14612            "class": exception_class,
14613            "message": error.to_string(),
14614            "properties": properties,
14615        }
14616    });
14617
14618    if let Some((sequence, activity_execution_id)) = failed_activity_boundary {
14619        command["failed_step_sequence"] = json!(sequence);
14620        command["failed_activity_execution_id"] = json!(activity_execution_id);
14621    }
14622
14623    command
14624}
14625
14626fn workflow_error_type(error: &Error) -> &'static str {
14627    match error {
14628        Error::ActivityFailed(failure) => match failure.kind {
14629            ActivityFailureKind::Failed => "ActivityFailed",
14630            ActivityFailureKind::Cancelled => "ActivityCancelled",
14631            ActivityFailureKind::TimedOut => "ActivityTimedOut",
14632        },
14633        Error::ChildWorkflowFailed(failure) => match failure.kind {
14634            ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14635            ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14636            ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14637        },
14638        Error::ParallelFailed(_) => "ParallelFailed",
14639        Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14640        Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14641        Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14642        _ => "RustWorkflowError",
14643    }
14644}
14645
14646fn workflow_error_non_retryable(error: &Error) -> bool {
14647    match error {
14648        Error::ActivityFailed(failure) => failure.non_retryable,
14649        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14650        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14651        Error::SagaCompensationFailed(failure) => {
14652            workflow_error_non_retryable(&failure.compensation_failure)
14653        }
14654        Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14655        _ => false,
14656    }
14657}
14658
14659fn workflow_task_integrity_error(error: &Error) -> bool {
14660    matches!(
14661        error,
14662        Error::NonDeterministicReplay(_)
14663            | Error::Protocol(_)
14664            | Error::MissingWorkflowCommandIdentity
14665            | Error::WorkflowStatePoisoned
14666    )
14667}
14668
14669fn decode_signal_event_arguments(
14670    event: &HistoryEvent,
14671    fallback_codec: &str,
14672) -> Result<Vec<AvroValue>> {
14673    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14674    validate_payload_codec(codec)?;
14675    let raw = signal_history_payload(&event.payload);
14676    let decoded = match raw.filter(|value| !value.is_null()) {
14677        Some(value) => decode_wire_avro_value(value, codec)?,
14678        None => AvroValue::Array(Vec::new()),
14679    };
14680    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14681        unreachable!("normalize_avro_arguments always returns an array");
14682    };
14683    Ok(arguments)
14684}
14685
14686fn decode_update_event_arguments(
14687    event: &HistoryEvent,
14688    fallback_codec: &str,
14689) -> Result<Vec<AvroValue>> {
14690    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14691    validate_payload_codec(codec)?;
14692    let decoded = match event
14693        .payload
14694        .get("arguments")
14695        .filter(|value| !value.is_null())
14696    {
14697        Some(value) => decode_wire_avro_value(value, codec)?,
14698        None => AvroValue::Array(Vec::new()),
14699    };
14700    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14701        unreachable!("normalize_avro_arguments always returns an array");
14702    };
14703    Ok(arguments)
14704}
14705
14706fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14707    let Some(export_events) = task
14708        .history_export
14709        .as_ref()
14710        .and_then(|export| export.get("history_events"))
14711        .and_then(Value::as_array)
14712    else {
14713        return Ok(());
14714    };
14715
14716    if export_events.len() > task.history_events.len() {
14717        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14718    }
14719
14720    Ok(())
14721}
14722
14723fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14724    let Some(export) = task.history_export.as_ref() else {
14725        return Ok(());
14726    };
14727    let signals = export
14728        .get("signals")
14729        .and_then(Value::as_array)
14730        .cloned()
14731        .unwrap_or_default();
14732    let activities = export
14733        .get("activities")
14734        .and_then(Value::as_array)
14735        .cloned()
14736        .unwrap_or_default();
14737    let export_codec = export
14738        .get("payloads")
14739        .and_then(|payloads| payloads.get("codec"))
14740        .and_then(Value::as_str)
14741        .unwrap_or(&task.payload_codec)
14742        .to_string();
14743    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14744
14745    for event in &mut task.history_events {
14746        if event.event_type == "ActivityCompleted" {
14747            let sequence = event
14748                .payload
14749                .get("sequence")
14750                .or_else(|| event.payload.get("workflow_sequence"))
14751                .and_then(value_as_u64);
14752            let Some(activity) = sequence.and_then(|sequence| {
14753                activities.iter().find(|activity| {
14754                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14755                })
14756            }) else {
14757                continue;
14758            };
14759            let Some(payload) = event.payload.as_object_mut() else {
14760                continue;
14761            };
14762            if missing_payload(payload.get("result")) {
14763                if let Some(result) = activity
14764                    .get("result")
14765                    .filter(|value| !missing_payload(Some(value)))
14766                {
14767                    payload.insert("result".to_string(), result.clone());
14768                }
14769            }
14770            for field in ["payload_codec", "activity_type"] {
14771                if payload
14772                    .get(field)
14773                    .and_then(Value::as_str)
14774                    .unwrap_or_default()
14775                    .is_empty()
14776                {
14777                    if let Some(value) = activity.get(field) {
14778                        payload.insert(field.to_string(), value.clone());
14779                    }
14780                }
14781            }
14782            continue;
14783        }
14784
14785        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14786            continue;
14787        }
14788        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14789        let command_id = event
14790            .payload
14791            .get("workflow_command_id")
14792            .or_else(|| event.raw.get("workflow_command_id"))
14793            .and_then(Value::as_str);
14794        let signal_name = event
14795            .payload
14796            .get("signal_name")
14797            .and_then(Value::as_str)
14798            .unwrap_or_default()
14799            .to_string();
14800        let matched = signals
14801            .iter()
14802            .find(|signal| {
14803                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14804            })
14805            .or_else(|| {
14806                signals.iter().find(|signal| {
14807                    command_id.is_some()
14808                        && signal.get("command_id").and_then(Value::as_str) == command_id
14809                })
14810            })
14811            .or_else(|| {
14812                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14813                let signal = signals
14814                    .iter()
14815                    .filter(|signal| {
14816                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14817                    })
14818                    .nth(*offset);
14819                if signal.is_some() {
14820                    *offset += 1;
14821                }
14822                signal
14823            });
14824        let Some(signal) = matched else {
14825            continue;
14826        };
14827        let signal_codec = signal
14828            .get("payload_codec")
14829            .and_then(Value::as_str)
14830            .unwrap_or(&export_codec);
14831        let Some(payload) = event.payload.as_object_mut() else {
14832            continue;
14833        };
14834        if missing_payload(payload.get("arguments")) {
14835            if let Some(arguments) = signal
14836                .get("arguments")
14837                .filter(|value| !missing_payload(Some(value)))
14838            {
14839                let envelope = match arguments {
14840                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14841                    other => other.clone(),
14842                };
14843                payload.insert("arguments".to_string(), envelope);
14844            }
14845        }
14846        if payload
14847            .get("payload_codec")
14848            .and_then(Value::as_str)
14849            .unwrap_or_default()
14850            .is_empty()
14851        {
14852            payload.insert("payload_codec".to_string(), json!(signal_codec));
14853        }
14854    }
14855
14856    Ok(())
14857}
14858
14859fn missing_payload(value: Option<&Value>) -> bool {
14860    match value {
14861        None | Some(Value::Null) => true,
14862        Some(Value::String(value)) => value.is_empty(),
14863        Some(_) => false,
14864    }
14865}
14866
14867fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14868    let export_signals = task
14869        .history_export
14870        .as_ref()
14871        .and_then(|export| export.get("signals"))
14872        .and_then(Value::as_array)
14873        .cloned()
14874        .unwrap_or_default();
14875    let export_codec = task
14876        .history_export
14877        .as_ref()
14878        .and_then(|export| export.get("payloads"))
14879        .and_then(|payloads| payloads.get("codec"))
14880        .and_then(Value::as_str)
14881        .unwrap_or(&task.payload_codec);
14882    let mut name_offsets: HashMap<String, usize> = HashMap::new();
14883    let mut signals = Vec::new();
14884
14885    for event in &task.history_events {
14886        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14887            continue;
14888        }
14889
14890        let name = event
14891            .payload
14892            .get("signal_name")
14893            .and_then(Value::as_str)
14894            .unwrap_or_default();
14895        if name.is_empty() {
14896            continue;
14897        }
14898        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14899        let command_id = event
14900            .payload
14901            .get("workflow_command_id")
14902            .or_else(|| event.raw.get("workflow_command_id"))
14903            .and_then(Value::as_str);
14904        let matched_export = export_signals
14905            .iter()
14906            .find(|candidate| {
14907                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14908            })
14909            .or_else(|| {
14910                export_signals.iter().find(|candidate| {
14911                    command_id.is_some()
14912                        && candidate.get("command_id").and_then(Value::as_str) == command_id
14913                })
14914            })
14915            .or_else(|| {
14916                let offset = name_offsets.entry(name.to_string()).or_default();
14917                let candidate = export_signals
14918                    .iter()
14919                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14920                    .nth(*offset);
14921                if candidate.is_some() {
14922                    *offset += 1;
14923                }
14924                candidate
14925            });
14926        let codec = event
14927            .payload
14928            .get("payload_codec")
14929            .and_then(Value::as_str)
14930            .or_else(|| {
14931                matched_export
14932                    .and_then(|signal| signal.get("payload_codec"))
14933                    .and_then(Value::as_str)
14934            })
14935            .unwrap_or(export_codec);
14936        let raw_arguments = signal_history_payload(&event.payload)
14937            .filter(|value| !value.is_null())
14938            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14939        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14940        let workflow_sequence = event
14941            .payload
14942            .get("workflow_sequence")
14943            .and_then(value_as_u64)
14944            .or_else(|| {
14945                matched_export
14946                    .and_then(|signal| signal.get("workflow_sequence"))
14947                    .and_then(value_as_u64)
14948            });
14949
14950        signals.push(QuerySignal {
14951            id: signal_id.map(str::to_string).or_else(|| {
14952                matched_export
14953                    .and_then(|signal| signal.get("id"))
14954                    .and_then(Value::as_str)
14955                    .map(str::to_string)
14956            }),
14957            name: name.to_string(),
14958            arguments,
14959            avro_arguments,
14960            workflow_sequence,
14961        });
14962    }
14963
14964    if signals.is_empty() {
14965        for signal in export_signals {
14966            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14967                continue;
14968            }
14969            let Some(name) = signal.get("name").and_then(Value::as_str) else {
14970                continue;
14971            };
14972            let codec = signal
14973                .get("payload_codec")
14974                .and_then(Value::as_str)
14975                .unwrap_or(export_codec);
14976            let (arguments, avro_arguments) =
14977                decode_query_signal_arguments(signal.get("arguments"), codec)?;
14978            signals.push(QuerySignal {
14979                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14980                name: name.to_string(),
14981                arguments,
14982                avro_arguments,
14983                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14984            });
14985        }
14986        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14987    }
14988
14989    Ok(signals)
14990}
14991
14992fn decode_query_signal_arguments(
14993    raw: Option<&Value>,
14994    codec: &str,
14995) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14996    validate_payload_codec(codec)?;
14997    let decoded = match raw.filter(|value| !value.is_null()) {
14998        Some(value) => decode_wire_avro_value(value, codec)?,
14999        None => AvroValue::Array(Vec::new()),
15000    };
15001    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
15002        unreachable!("normalize_avro_arguments always returns an array");
15003    };
15004    let arguments = avro_arguments
15005        .iter()
15006        .cloned()
15007        .map(AvroValue::into_json)
15008        .collect::<Result<Vec<_>>>()?;
15009    Ok((arguments, avro_arguments))
15010}
15011
15012fn value_as_u64(value: &Value) -> Option<u64> {
15013    value
15014        .as_u64()
15015        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
15016}
15017
15018#[cfg(test)]
15019mod tests {
15020    use super::*;
15021    mod runtime_payloads;
15022    mod runtime_uploads;
15023    use std::{
15024        fs,
15025        io::{Read, Write},
15026        net::{SocketAddr, TcpListener, TcpStream},
15027        process::Command as ProcessCommand,
15028        sync::atomic::AtomicUsize,
15029        thread,
15030    };
15031
15032    #[derive(Clone, Copy, Debug)]
15033    enum InvalidTaskPayloadCodec {
15034        Missing,
15035        Null,
15036        NonString,
15037    }
15038
15039    impl InvalidTaskPayloadCodec {
15040        fn label(self) -> &'static str {
15041            match self {
15042                Self::Missing => "missing",
15043                Self::Null => "null",
15044                Self::NonString => "non-string",
15045            }
15046        }
15047
15048        fn apply(self, task: &mut Value) {
15049            let task = task.as_object_mut().expect("task fixture object");
15050            match self {
15051                Self::Missing => {
15052                    task.remove("payload_codec");
15053                }
15054                Self::Null => {
15055                    task.insert("payload_codec".to_string(), Value::Null);
15056                }
15057                Self::NonString => {
15058                    task.insert("payload_codec".to_string(), json!(42));
15059                }
15060            }
15061        }
15062    }
15063
15064    fn fixture_envelope(value: Value) -> Value {
15065        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
15066    }
15067
15068    fn fixture_blob(value: Value) -> String {
15069        encode_payload(&value, DEFAULT_CODEC)
15070            .expect("encode Avro test fixture")
15071            .blob
15072    }
15073
15074    #[test]
15075    fn client_builder_rejects_the_sdk_owned_api_suffix() {
15076        for base_url in [
15077            "http://127.0.0.1:8080/api",
15078            "http://localhost:8080/api/",
15079            "https://runtime.example.test/namespaces/orders/api",
15080        ] {
15081            let error = Client::builder(base_url)
15082                .build()
15083                .expect_err("SDK-owned /api suffix must be rejected during build");
15084
15085            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
15086            assert!(
15087                error.to_string().contains("SDK appends /api automatically"),
15088                "the validation error must explain how to fix the endpoint"
15089            );
15090        }
15091    }
15092
15093    #[test]
15094    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
15095        for (base_url, expected) in [
15096            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
15097            (
15098                "http://localhost:8080/durable-workflow/",
15099                "http://localhost:8080/durable-workflow",
15100            ),
15101            (
15102                "https://runtime.example.test/namespaces/orders",
15103                "https://runtime.example.test/namespaces/orders",
15104            ),
15105            (
15106                "https://runtime.example.test/gateway/api/namespaces/orders",
15107                "https://runtime.example.test/gateway/api/namespaces/orders",
15108            ),
15109            (
15110                "https://api.example.test/runtime/orders/",
15111                "https://api.example.test/runtime/orders",
15112            ),
15113        ] {
15114            let client = Client::builder(base_url)
15115                .build()
15116                .expect("Server and Cloud runtime base URL must remain valid");
15117
15118            assert_eq!(client.base_url, expected);
15119        }
15120    }
15121
15122    #[test]
15123    fn workflow_completion_uses_the_additive_command_protocol_floor() {
15124        assert_eq!(
15125            workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
15126            WORKER_PROTOCOL_VERSION
15127        );
15128        assert_eq!(
15129            workflow_completion_protocol_version(&[json!({
15130                "type": "upsert_search_attributes",
15131                "attributes": {"OrderStatus": "waiting"},
15132            })]),
15133            SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
15134        );
15135        assert_eq!(
15136            workflow_completion_protocol_version(&[json!({
15137                "type": "upsert_search_attributes",
15138                "attributes": {"OrderStatus": "waiting"},
15139                "attribute_types": {"OrderStatus": "keyword"},
15140            })]),
15141            TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
15142        );
15143        assert_eq!(
15144            workflow_completion_protocol_version(&[
15145                json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
15146                json!({"type": "open_condition_wait", "condition_key": "ready"}),
15147            ]),
15148            MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
15149        );
15150        assert_eq!(
15151            workflow_completion_protocol_version(&[
15152                json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
15153                json!({"type": "open_condition_wait", "condition_key": "ready"}),
15154            ]),
15155            CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
15156        );
15157        assert_eq!(
15158            workflow_completion_protocol_version(&[json!({
15159                "type": "open_condition_wait",
15160                "condition_wait_occurrence_id": "rust:condition-wait:0",
15161                "condition_key": "ready",
15162            })]),
15163            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
15164        );
15165        assert_eq!(
15166            workflow_completion_protocol_version_with_message_streams(
15167                &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
15168                true,
15169            ),
15170            MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
15171        );
15172        assert_eq!(
15173            workflow_completion_protocol_version_with_message_streams(
15174                &[json!({
15175                    "type": "open_condition_wait",
15176                    "condition_wait_occurrence_id": "rust:condition-wait:0",
15177                    "condition_key": "ready",
15178                })],
15179                true,
15180            ),
15181            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
15182        );
15183    }
15184
15185    #[test]
15186    fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
15187        let manifest = portable_worker_affinity_capability_manifest();
15188
15189        for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
15190            assert_eq!(manifest[capability]["supported"], json!(false));
15191            assert_eq!(
15192                manifest[capability]["minimum_protocol_version"],
15193                json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
15194            );
15195            assert!(manifest[capability]["reason"]
15196                .as_str()
15197                .is_some_and(|reason| !reason.is_empty()));
15198        }
15199    }
15200
15201    fn typed_fidelity_probe() -> AvroValue {
15202        AvroValue::Map(BTreeMap::from([
15203            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
15204            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
15205            (
15206                "numeric".to_string(),
15207                AvroValue::Map(BTreeMap::from([
15208                    ("0".to_string(), AvroValue::String("zero".to_string())),
15209                    ("1".to_string(), AvroValue::String("one".to_string())),
15210                ])),
15211            ),
15212            (
15213                "nested".to_string(),
15214                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
15215                    "enabled".to_string(),
15216                    AvroValue::Boolean(true),
15217                )]))]),
15218            ),
15219            (
15220                "projection_collisions".to_string(),
15221                AvroValue::Array(projection_collision_probe()),
15222            ),
15223        ]))
15224    }
15225
15226    fn projection_collision_probe() -> Vec<AvroValue> {
15227        vec![
15228            AvroValue::Map(BTreeMap::from([
15229                ("$type".to_string(), AvroValue::String("bytes".to_string())),
15230                (
15231                    "base64".to_string(),
15232                    AvroValue::String("ordinary user text".to_string()),
15233                ),
15234            ])),
15235            AvroValue::Map(BTreeMap::from([
15236                ("$type".to_string(), AvroValue::String("map".to_string())),
15237                (
15238                    "entries".to_string(),
15239                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
15240                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
15241                        (
15242                            "value".to_string(),
15243                            AvroValue::String("user map".to_string()),
15244                        ),
15245                    ]))]),
15246                ),
15247            ])),
15248        ]
15249    }
15250
15251    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15252    struct TypedContract {
15253        nested: TypedNested,
15254        mode: TypedMode,
15255        optional: Option<String>,
15256        absent: Option<String>,
15257        items: Vec<i64>,
15258        labels: BTreeMap<String, String>,
15259        bytes: serde_bytes::ByteBuf,
15260        signed: i64,
15261        finite: f64,
15262    }
15263
15264    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15265    struct TypedNested {
15266        enabled: bool,
15267    }
15268
15269    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15270    enum TypedMode {
15271        Detailed { label: String },
15272    }
15273
15274    fn typed_contract() -> TypedContract {
15275        TypedContract {
15276            nested: TypedNested { enabled: true },
15277            mode: TypedMode::Detailed {
15278                label: "compiler-checked".to_string(),
15279            },
15280            optional: Some("present".to_string()),
15281            absent: None,
15282            items: vec![i64::MIN, 0, i64::MAX],
15283            labels: BTreeMap::from([
15284                ("language".to_string(), "rust".to_string()),
15285                ("wire".to_string(), "avro".to_string()),
15286            ]),
15287            bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
15288            signed: -9_223_372_036_854_775_000,
15289            finite: 12.5,
15290        }
15291    }
15292
15293    #[derive(Clone, Debug, Default, PartialEq)]
15294    struct ReplayCounterState {
15295        loaded: Option<String>,
15296        count: i64,
15297        finished: bool,
15298    }
15299
15300    fn replay_counter_worker() -> Worker {
15301        let client = Client::new("http://127.0.0.1:8080").expect("client");
15302        let mut worker = Worker::new(client, "rust-workers");
15303        worker.register_replayed_workflow(
15304            "replay-counter",
15305            ReplayCounterState::default,
15306            |ctx, _input, state| async move {
15307                let loaded = ctx.activity("load-counter", json!([])).await?;
15308                state.update(|current| {
15309                    current.loaded = loaded.as_str().map(str::to_string);
15310                })?;
15311                for _ in 0..2 {
15312                    let signal = ctx.wait_signal("increment").await?;
15313                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15314                    state.update(|current| current.count += amount)?;
15315                }
15316                state.update(|current| current.finished = true)?;
15317                state.read(|current| Ok(json!(current.count)))?
15318            },
15319        );
15320        worker.register_replayed_query::<ReplayCounterState, _, _>(
15321            "replay-counter",
15322            "current",
15323            |_ctx, state, _args| async move {
15324                Ok(json!({
15325                    "loaded": state.loaded,
15326                    "count": state.count,
15327                    "finished": state.finished,
15328                }))
15329            },
15330        );
15331        worker.register_replayed_query::<ReplayCounterState, _, _>(
15332            "replay-counter",
15333            "detached-mutation",
15334            |_ctx, state, _args| async move {
15335                let mut detached = (*state).clone();
15336                detached.count = 999;
15337                Ok(json!(detached.count))
15338            },
15339        );
15340        worker.register_replayed_query::<ReplayCounterState, _, _>(
15341            "replay-counter",
15342            "failed-mutation",
15343            |_ctx, state, _args| async move {
15344                let mut detached = (*state).clone();
15345                detached.count = 999;
15346                Err(Error::WorkerLoop("query refused".to_string()))
15347            },
15348        );
15349        worker
15350    }
15351
15352    fn replay_counter_query(
15353        query_name: &str,
15354        history_events: Value,
15355        run_status: &str,
15356    ) -> QueryTask {
15357        let arguments = fixture_envelope(json!([]));
15358        serde_json::from_value(json!({
15359            "query_task_id": format!("query-{query_name}"),
15360            "workflow_type": "replay-counter",
15361            "query_name": query_name,
15362            "payload_codec": DEFAULT_CODEC,
15363            "workflow_arguments": arguments.clone(),
15364            "query_arguments": arguments,
15365            "history_events": history_events,
15366            "run_status": run_status,
15367        }))
15368        .expect("query task")
15369    }
15370
15371    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15372        workflow_context_with_codec(history, DEFAULT_CODEC)
15373    }
15374
15375    fn workflow_context_with_codec(
15376        history: Vec<HistoryEvent>,
15377        payload_codec: &str,
15378    ) -> WorkflowContext {
15379        WorkflowContext {
15380            state: Arc::new(Mutex::new(
15381                WorkflowState::new_with_identity(
15382                    history,
15383                    None,
15384                    None,
15385                    "rust-workers".to_string(),
15386                    payload_codec.to_string(),
15387                    None,
15388                )
15389                .expect("valid workflow history"),
15390            )),
15391        }
15392    }
15393
15394    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15395        HistoryEvent {
15396            event_type: event_type.to_string(),
15397            payload,
15398            raw: HashMap::new(),
15399        }
15400    }
15401
15402    fn parallel_path_entry(
15403        kind: &str,
15404        base: u64,
15405        size: usize,
15406        index: usize,
15407    ) -> ParallelGroupMetadata {
15408        parallel_group_entry(base, size, index, kind)
15409    }
15410
15411    fn parallel_history_event(
15412        event_type: &str,
15413        sequence: u64,
15414        identity_field: &str,
15415        identity: &str,
15416        path: Vec<ParallelGroupMetadata>,
15417        result: Option<Value>,
15418    ) -> HistoryEvent {
15419        let mut payload = serde_json::Map::from_iter([
15420            ("sequence".to_string(), json!(sequence)),
15421            (identity_field.to_string(), json!(identity)),
15422        ]);
15423        let inner = path.last().expect("parallel history path");
15424        apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15425        payload.insert("parallel_group_path".to_string(), json!(path));
15426        if let Some(result) = result {
15427            let field = if event_type == "ChildRunCompleted" {
15428                "result"
15429            } else {
15430                "result"
15431            };
15432            payload.insert(field.to_string(), fixture_envelope(result));
15433            payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15434        }
15435        history_event(event_type, Value::Object(payload))
15436    }
15437
15438    fn nested_parallel_operations() -> Vec<ParallelOperation> {
15439        vec![
15440            ParallelOperation::activity("first", json!([])),
15441            ParallelOperation::group(vec![
15442                ParallelOperation::child_workflow(
15443                    "second",
15444                    ChildWorkflowOptions::new("child-workers"),
15445                    json!([]),
15446                ),
15447                ParallelOperation::activity("third", json!([])),
15448            ]),
15449        ]
15450    }
15451
15452    fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15453        let outer = [
15454            parallel_path_entry("mixed", 1, 3, 0),
15455            parallel_path_entry("mixed", 1, 3, 1),
15456            parallel_path_entry("mixed", 1, 3, 2),
15457        ];
15458        [
15459            vec![outer[0].clone()],
15460            vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15461            vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15462        ]
15463    }
15464
15465    #[test]
15466    fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15467        let ctx = workflow_context(Vec::new());
15468        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15469        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15470
15471        assert!(matches!(
15472            call.as_mut().poll(&mut task_context),
15473            Poll::Pending
15474        ));
15475        let commands = ctx.take_commands().expect("parallel commands");
15476        assert_eq!(
15477            commands
15478                .iter()
15479                .map(|command| command["type"].as_str().unwrap_or_default())
15480                .collect::<Vec<_>>(),
15481            [
15482                "schedule_activity",
15483                "start_child_workflow",
15484                "schedule_activity"
15485            ]
15486        );
15487        let paths = nested_parallel_paths();
15488        for (command, path) in commands.iter().zip(paths) {
15489            assert_eq!(command["parallel_group_path"], json!(path));
15490            assert_eq!(
15491                command["parallel_group_id"],
15492                json!(path.last().expect("inner group").parallel_group_id)
15493            );
15494        }
15495    }
15496
15497    fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15498        let paths = nested_parallel_paths();
15499        let third = parallel_history_event(
15500            "ActivityCompleted",
15501            3,
15502            "activity_type",
15503            "third",
15504            paths[2].clone(),
15505            Some(json!("three")),
15506        );
15507        vec![
15508            parallel_history_event(
15509                "ActivityCompleted",
15510                1,
15511                "activity_type",
15512                "first",
15513                paths[0].clone(),
15514                Some(json!("one")),
15515            ),
15516            parallel_history_event(
15517                "ChildWorkflowScheduled",
15518                2,
15519                "child_workflow_type",
15520                "second",
15521                paths[1].clone(),
15522                None,
15523            ),
15524            parallel_history_event(
15525                "ChildRunCompleted",
15526                2,
15527                "child_workflow_type",
15528                "second",
15529                paths[1].clone(),
15530                Some(json!("two")),
15531            ),
15532            third.clone(),
15533            third,
15534        ]
15535    }
15536
15537    #[test]
15538    fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15539        for _restart_or_completed_replay in 0..2 {
15540            let ctx = workflow_context(completed_nested_parallel_history());
15541            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15542            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15543            let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15544                panic!("completed nested parallel history must replay");
15545            };
15546            assert_eq!(
15547                results,
15548                vec![
15549                    ParallelResult::Activity(json!("one")),
15550                    ParallelResult::Group(vec![
15551                        ParallelResult::ChildWorkflow(ChildWorkflowResult {
15552                            parent: WorkflowIdentity {
15553                                workflow_id: None,
15554                                run_id: None,
15555                            },
15556                            child: WorkflowIdentity {
15557                                workflow_id: None,
15558                                run_id: None,
15559                            },
15560                            child_workflow_type: Some("second".to_string()),
15561                            result: json!("two"),
15562                        }),
15563                        ParallelResult::Activity(json!("three")),
15564                    ]),
15565                ]
15566            );
15567            assert!(ctx.take_commands().expect("commands").is_empty());
15568            ctx.ensure_history_consumed().expect("history consumed");
15569        }
15570    }
15571
15572    #[test]
15573    fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15574        let paths = nested_parallel_paths();
15575        let history = vec![
15576            parallel_history_event(
15577                "ActivityCompleted",
15578                1,
15579                "activity_type",
15580                "first",
15581                paths[0].clone(),
15582                Some(json!("one")),
15583            ),
15584            parallel_history_event(
15585                "ChildWorkflowScheduled",
15586                2,
15587                "child_workflow_type",
15588                "second",
15589                paths[1].clone(),
15590                None,
15591            ),
15592            parallel_history_event(
15593                "ChildRunFailed",
15594                2,
15595                "child_workflow_type",
15596                "second",
15597                paths[1].clone(),
15598                None,
15599            ),
15600            parallel_history_event(
15601                "ActivityCompleted",
15602                3,
15603                "activity_type",
15604                "third",
15605                paths[2].clone(),
15606                Some(json!("late")),
15607            ),
15608        ];
15609        let ctx = workflow_context(history);
15610        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15611        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15612        let outcome = call.as_mut().poll(&mut task_context);
15613        let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15614            panic!("one failed child must return a typed partial failure: {outcome:?}");
15615        };
15616        assert_eq!(failure.member_path, [1, 0]);
15617        assert_eq!(failure.group_id, "parallel-calls:1:3");
15618        assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15619        assert_eq!(
15620            failure
15621                .completed
15622                .iter()
15623                .map(|completion| completion.member_path.clone())
15624                .collect::<Vec<_>>(),
15625            [vec![0], vec![1, 1]]
15626        );
15627    }
15628
15629    #[test]
15630    fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15631        let paths = nested_parallel_paths();
15632        let history = vec![
15633            parallel_history_event(
15634                "ActivityScheduled",
15635                1,
15636                "activity_type",
15637                "first",
15638                paths[0].clone(),
15639                None,
15640            ),
15641            parallel_history_event(
15642                "ChildWorkflowScheduled",
15643                2,
15644                "child_workflow_type",
15645                "second",
15646                paths[1].clone(),
15647                None,
15648            ),
15649            parallel_history_event(
15650                "ActivityScheduled",
15651                3,
15652                "activity_type",
15653                "third",
15654                paths[2].clone(),
15655                None,
15656            ),
15657        ];
15658        for _restart in 0..2 {
15659            let ctx = workflow_context(history.clone());
15660            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15661            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15662            let outcome = call.as_mut().poll(&mut task_context);
15663            assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15664            assert!(ctx.take_commands().expect("commands").is_empty());
15665        }
15666    }
15667
15668    fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15669        vec![selection_group_entry(
15670            1,
15671            2,
15672            index,
15673            "activity",
15674            &SelectionMemberMetadata {
15675                key: SelectionKey::Name(key.to_string()),
15676                index,
15677                base_sequence: index as u64 + 1,
15678                size: 1,
15679                kind: "activity".to_string(),
15680            },
15681        )]
15682    }
15683
15684    fn selection_activity_event(
15685        event_type: &str,
15686        index: usize,
15687        key: &str,
15688        result: Option<Value>,
15689    ) -> HistoryEvent {
15690        let sequence = index as u64 + 1;
15691        let mut event = parallel_history_event(
15692            event_type,
15693            sequence,
15694            "activity_type",
15695            &format!("{key}-activity"),
15696            selection_path(index, key),
15697            result,
15698        );
15699        event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15700        event.raw.insert(
15701            "id".to_string(),
15702            json!(if event_type == "ActivityCompleted" {
15703                format!("event-{key}")
15704            } else {
15705                format!("{event_type}-{key}")
15706            }),
15707        );
15708        event
15709    }
15710
15711    fn selection_winner_marker() -> HistoryEvent {
15712        history_event(
15713            "SelectionResolved",
15714            json!({
15715                "selection_group_id": "select-calls:1:2",
15716                "selection_group_base_sequence": 1,
15717                "selection_group_size": 2,
15718                "member_key": "fast",
15719                "member_index": 1,
15720                "member_base_sequence": 2,
15721                "member_size": 1,
15722                "operation_kind": "activity",
15723                "operation_identity": "activity-fast",
15724                "outcome": "completed",
15725                "resolution_event_id": "event-fast",
15726                "resolution_event_type": "ActivityCompleted",
15727            }),
15728        )
15729    }
15730
15731    fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15732        ctx.select_keyed(vec![
15733            (
15734                "slow",
15735                ParallelOperation::activity_with_options(
15736                    "slow-activity",
15737                    ActivityOptions::new().task_queue("default"),
15738                    json!([]),
15739                ),
15740            ),
15741            (
15742                "fast",
15743                ParallelOperation::activity_with_options(
15744                    "fast-activity",
15745                    ActivityOptions::new().task_queue("default"),
15746                    json!([]),
15747                ),
15748            ),
15749        ])
15750    }
15751
15752    fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15753        let ctx = workflow_context(history);
15754        let mut call = Box::pin(keyed_activity_selection(&ctx));
15755        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15756        let selected = match call.as_mut().poll(&mut task_context) {
15757            Poll::Ready(Ok(selected)) => selected,
15758            Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15759            Poll::Pending => panic!("persisted selection winner must replay without pending"),
15760        };
15761        assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15762        assert_eq!(
15763            selected.value,
15764            Some(ParallelResult::Activity(json!("winner-value")))
15765        );
15766        let slow = selected
15767            .handle(&SelectionKey::Name("slow".to_string()))
15768            .expect("slow handle")
15769            .clone();
15770        let mut await_slow = Box::pin(slow.await_result());
15771        assert!(matches!(
15772            await_slow.as_mut().poll(&mut task_context),
15773            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15774        ));
15775        assert!(ctx.take_commands().expect("commands").is_empty());
15776    }
15777
15778    const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15779
15780    fn canonical_selection_history() -> Vec<HistoryEvent> {
15781        const FIXTURE: &[u8] =
15782            include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15783        assert_eq!(
15784            format!("{:x}", Sha256::digest(FIXTURE)),
15785            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15786        );
15787        let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15788
15789        serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15790    }
15791
15792    #[test]
15793    fn selection_fresh_process_entrypoint() {
15794        let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15795            return;
15796        };
15797        let persisted = fs::read(path).expect("persisted selection history");
15798        assert_eq!(
15799            format!("{:x}", Sha256::digest(&persisted)),
15800            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15801        );
15802        let fixture: Value =
15803            serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15804        let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15805            .expect("valid persisted selection history");
15806
15807        assert_persisted_selection_replay(history);
15808    }
15809
15810    #[test]
15811    fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15812        let ctx = workflow_context(Vec::new());
15813        let mut call = Box::pin(keyed_activity_selection(&ctx));
15814        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15815
15816        assert!(matches!(
15817            call.as_mut().poll(&mut task_context),
15818            Poll::Pending
15819        ));
15820        let commands = ctx.take_commands().expect("selection commands");
15821        assert_eq!(commands.len(), 2);
15822        assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15823        assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15824        assert!(commands.iter().all(|command| {
15825            command["parallel_group_id"] == json!("select-calls:1:2")
15826                && command["parallel_group_mode"] == json!("select")
15827        }));
15828    }
15829
15830    #[test]
15831    fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15832        let ctx = workflow_context(Vec::new());
15833        let mut invalid = Box::pin(ctx.select_keyed(vec![(
15834            "",
15835            ParallelOperation::activity("invalid", json!([])),
15836        )]));
15837        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15838        assert!(matches!(
15839            invalid.as_mut().poll(&mut task_context),
15840            Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15841                reason: "selection_key_invalid",
15842                ..
15843            })))
15844        ));
15845
15846        for invalid_key in [json!(""), json!(-1)] {
15847            let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15848            event.payload["selection_member_key"] = invalid_key.clone();
15849            event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15850            assert!(matches!(
15851                WorkflowState::new_with_identity(
15852                    vec![event],
15853                    None,
15854                    None,
15855                    "rust-workers".to_string(),
15856                    DEFAULT_CODEC.to_string(),
15857                    None,
15858                ),
15859                Err(Error::NonDeterministicReplay(_))
15860            ));
15861        }
15862    }
15863
15864    #[test]
15865    fn selection_preserves_valid_named_and_numeric_keys() {
15866        let ctx = workflow_context(Vec::new());
15867        let mut selection = Box::pin(ctx.select_keyed(vec![
15868            (
15869                SelectionKey::Index(0),
15870                ParallelOperation::activity("numeric", json!([])),
15871            ),
15872            (
15873                SelectionKey::Name("named".to_string()),
15874                ParallelOperation::timer(Duration::from_secs(1)),
15875            ),
15876        ]));
15877        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15878
15879        assert!(matches!(
15880            selection.as_mut().poll(&mut task_context),
15881            Poll::Pending
15882        ));
15883        let commands = ctx.take_commands().expect("selection commands");
15884        assert_eq!(commands[0]["selection_member_key"], json!(0));
15885        assert_eq!(commands[1]["selection_member_key"], json!("named"));
15886    }
15887
15888    #[test]
15889    fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15890        let history = canonical_selection_history();
15891        assert_persisted_selection_replay(history.clone());
15892
15893        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15894            .join("tests/fixtures/durable_selection_runtime_history.json");
15895        let output =
15896            ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15897                .args([
15898                    "--exact",
15899                    "tests::selection_fresh_process_entrypoint",
15900                    "--nocapture",
15901                ])
15902                .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15903                .output()
15904                .expect("run fresh selection replay process");
15905
15906        assert!(
15907            output.status.success(),
15908            "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15909            String::from_utf8_lossy(&output.stdout),
15910            String::from_utf8_lossy(&output.stderr),
15911        );
15912    }
15913
15914    #[test]
15915    fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15916        let mut history = canonical_selection_history();
15917        history.retain(|event| event.event_type != "SelectionResolved");
15918        let ctx = workflow_context(history);
15919        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15920        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15921
15922        assert!(matches!(
15923            selection.as_mut().poll(&mut task_context),
15924            Poll::Pending
15925        ));
15926        assert!(ctx.take_commands().expect("commands").is_empty());
15927        assert!(
15928            ctx.matched_recorded_pending()
15929                .expect("selection pending state"),
15930            "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15931        );
15932    }
15933
15934    #[test]
15935    fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15936        for (terminal_event, predicate_satisfied, timeout_seconds) in [
15937            ("ConditionWaitSatisfied", true, None),
15938            ("ConditionWaitTimedOut", false, Some(0)),
15939        ] {
15940            let member = SelectionMemberMetadata {
15941                key: SelectionKey::Name("condition".to_string()),
15942                index: 0,
15943                base_sequence: 1,
15944                size: 1,
15945                kind: "condition".to_string(),
15946            };
15947            let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15948            let mut payload = json!({
15949                "sequence": 1,
15950                "condition_wait_id": "condition-1",
15951                "condition_wait_occurrence_id": "rust:condition-wait:0",
15952                "condition_key": "ready",
15953                "condition_definition_fingerprint": "sha256:ready-v1",
15954                "parallel_group_path": path,
15955            });
15956            payload
15957                .as_object_mut()
15958                .expect("condition history payload")
15959                .extend(
15960                    serde_json::to_value(&path[0])
15961                        .expect("condition selection metadata")
15962                        .as_object()
15963                        .expect("condition selection metadata object")
15964                        .clone(),
15965                );
15966            if let Some(timeout_seconds) = timeout_seconds {
15967                payload["timeout_seconds"] = json!(timeout_seconds);
15968            }
15969            let history = vec![
15970                history_event("ConditionWaitOpened", payload.clone()),
15971                history_event(terminal_event, payload),
15972            ];
15973            let ctx = workflow_context(history);
15974            let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15975            if timeout_seconds.is_some() {
15976                options = options.timeout(Duration::ZERO);
15977            }
15978            let mut selection = Box::pin(ctx.select_keyed(vec![(
15979                "condition",
15980                ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15981            )]));
15982            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15983
15984            assert!(matches!(
15985                selection.as_mut().poll(&mut task_context),
15986                Poll::Pending
15987            ));
15988            assert!(ctx.take_commands().expect("commands").is_empty());
15989            assert!(
15990                ctx.matched_recorded_pending()
15991                    .expect("condition selection pending state"),
15992                "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15993            );
15994        }
15995    }
15996
15997    #[test]
15998    fn selection_immediate_condition_members_open_a_durable_wait() {
15999        for predicate_satisfied in [true, false] {
16000            let ctx = workflow_context(Vec::new());
16001            let mut selection = Box::pin(ctx.select_keyed(vec![(
16002                "condition",
16003                ParallelOperation::condition(
16004                    ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
16005                    move || Ok(predicate_satisfied),
16006                ),
16007            )]));
16008            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16009
16010            assert!(matches!(
16011                selection.as_mut().poll(&mut task_context),
16012                Poll::Pending
16013            ));
16014            let commands = ctx.take_commands().expect("condition selection command");
16015            assert_eq!(commands.len(), 1);
16016            assert_eq!(commands[0]["type"], json!("open_condition_wait"));
16017            assert_eq!(commands[0]["timeout_seconds"], json!(0));
16018            assert_eq!(
16019                commands[0]["parallel_group_path"][0]["parallel_group_mode"],
16020                json!("select")
16021            );
16022        }
16023    }
16024
16025    #[test]
16026    fn selection_loser_cancellation_is_explicit_and_idempotent() {
16027        let history = vec![
16028            selection_activity_event("ActivityScheduled", 0, "slow", None),
16029            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16030            selection_winner_marker(),
16031        ];
16032        let ctx = workflow_context(history.clone());
16033        let mut call = Box::pin(keyed_activity_selection(&ctx));
16034        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16035        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
16036            panic!("winner must replay");
16037        };
16038        let slow = selected
16039            .handle(&SelectionKey::Name("slow".to_string()))
16040            .expect("slow handle")
16041            .clone();
16042        let mut cancel = Box::pin(slow.cancel());
16043        assert!(matches!(
16044            cancel.as_mut().poll(&mut task_context),
16045            Poll::Pending
16046        ));
16047        assert!(matches!(
16048            cancel.as_mut().poll(&mut task_context),
16049            Poll::Pending
16050        ));
16051        let commands = ctx.take_commands().expect("cancel command");
16052        assert_eq!(commands.len(), 1);
16053        assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
16054        assert_eq!(commands[0]["member_key"], json!("slow"));
16055
16056        let mut cancelled_history = history;
16057        cancelled_history.push(history_event(
16058            "SelectionOperationCancelled",
16059            json!({
16060                "selection_group_id": "select-calls:1:2",
16061                "member_key": "slow",
16062                "member_index": 0,
16063                "member_base_sequence": 1,
16064                "member_size": 1,
16065                "operation_kind": "activity",
16066                "operation_identity": "activity-slow",
16067                "cancelled_at": "2026-08-27T00:00:00Z",
16068            }),
16069        ));
16070        let replayed = workflow_context(cancelled_history);
16071        let mut call = Box::pin(keyed_activity_selection(&replayed));
16072        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
16073            panic!("winner must replay after cancellation");
16074        };
16075        let slow = selected
16076            .handle(&SelectionKey::Name("slow".to_string()))
16077            .expect("slow handle")
16078            .clone();
16079        let mut cancel = Box::pin(slow.cancel());
16080        assert!(matches!(
16081            cancel.as_mut().poll(&mut task_context),
16082            Poll::Ready(Ok(()))
16083        ));
16084        assert!(replayed.take_commands().expect("commands").is_empty());
16085    }
16086
16087    #[test]
16088    fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
16089        let base_history = vec![
16090            selection_activity_event("ActivityScheduled", 0, "slow", None),
16091            selection_activity_event("ActivityScheduled", 1, "fast", None),
16092            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16093            selection_winner_marker(),
16094        ];
16095        for (field, corrupt) in [
16096            ("member_key", json!("fast")),
16097            ("member_index", json!(1)),
16098            ("member_base_sequence", json!(3)),
16099            ("member_size", json!(2)),
16100            ("operation_kind", json!("timer")),
16101            ("operation_identity", json!("forged")),
16102        ] {
16103            let mut cancellation = json!({
16104                "selection_group_id": "select-calls:1:2",
16105                "member_key": "slow",
16106                "member_index": 0,
16107                "member_base_sequence": 1,
16108                "member_size": 1,
16109                "operation_kind": "activity",
16110                "operation_identity": "activity-slow",
16111            });
16112            cancellation[field] = corrupt;
16113            let mut history = base_history.clone();
16114            history.push(history_event("SelectionOperationCancelled", cancellation));
16115            let ctx = workflow_context(history);
16116            let mut selection = Box::pin(keyed_activity_selection(&ctx));
16117            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16118
16119            assert!(matches!(
16120                selection.as_mut().poll(&mut task_context),
16121                Poll::Ready(Err(Error::NonDeterministicReplay(_)))
16122            ));
16123        }
16124    }
16125
16126    #[test]
16127    fn selection_child_identity_prefers_the_durable_run_id() {
16128        let ctx = workflow_context(vec![history_event(
16129            "ChildWorkflowScheduled",
16130            json!({
16131                "sequence": 1,
16132                "child_workflow_type": "child",
16133                "child_workflow_instance_id": "child-instance",
16134                "child_workflow_run_id": "child-run",
16135            }),
16136        )]);
16137        let state = ctx.state.lock().expect("workflow state");
16138
16139        assert_eq!(
16140            selection_operation_identity(&state, "child", 1, 1),
16141            "child-run"
16142        );
16143    }
16144
16145    #[test]
16146    fn selection_activity_identity_requires_canonical_execution_id() {
16147        let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
16148        let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
16149        let mut fast_completed =
16150            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
16151        for event in [&mut fast_open, &mut fast_completed] {
16152            event
16153                .payload
16154                .as_object_mut()
16155                .expect("activity payload")
16156                .remove("activity_execution_id");
16157            event.payload["activity_id"] = json!("forged-activity-id");
16158        }
16159        let mut marker = selection_winner_marker();
16160        marker.payload["operation_identity"] = json!("forged-activity-id");
16161        let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
16162        let mut selection = Box::pin(keyed_activity_selection(&ctx));
16163        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16164
16165        assert!(matches!(
16166            selection.as_mut().poll(&mut task_context),
16167            Poll::Ready(Err(Error::NonDeterministicReplay(_)))
16168        ));
16169    }
16170
16171    #[test]
16172    fn selection_completion_before_cancellation_remains_awaitable() {
16173        let history = vec![
16174            selection_activity_event("ActivityScheduled", 0, "slow", None),
16175            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16176            selection_winner_marker(),
16177            selection_activity_event(
16178                "ActivityCompleted",
16179                0,
16180                "slow",
16181                Some(json!("completed-first")),
16182            ),
16183        ];
16184        let ctx = workflow_context(history);
16185        let mut selection = Box::pin(keyed_activity_selection(&ctx));
16186        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16187        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16188            panic!("winner must replay");
16189        };
16190        let slow = selected
16191            .handle(&SelectionKey::Name("slow".to_string()))
16192            .expect("slow handle")
16193            .clone();
16194        let mut cancel = Box::pin(slow.cancel());
16195        assert!(matches!(
16196            cancel.as_mut().poll(&mut task_context),
16197            Poll::Ready(Ok(()))
16198        ));
16199        let mut await_slow = Box::pin(slow.await_result());
16200        assert!(matches!(
16201            await_slow.as_mut().poll(&mut task_context),
16202            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
16203        ));
16204        let commands = ctx.take_commands().expect("commands");
16205        assert!(commands.is_empty());
16206    }
16207
16208    #[test]
16209    fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
16210        let nested_member = SelectionMemberMetadata {
16211            key: SelectionKey::Name("nested".to_string()),
16212            index: 0,
16213            base_sequence: 1,
16214            size: 2,
16215            kind: "group".to_string(),
16216        };
16217        let deadline_member = SelectionMemberMetadata {
16218            key: SelectionKey::Name("deadline".to_string()),
16219            index: 1,
16220            base_sequence: 3,
16221            size: 1,
16222            kind: "timer".to_string(),
16223        };
16224        let nested_paths = [
16225            vec![
16226                selection_group_entry(1, 3, 0, "mixed", &nested_member),
16227                parallel_group_entry(1, 2, 0, "activity"),
16228            ],
16229            vec![
16230                selection_group_entry(1, 3, 1, "mixed", &nested_member),
16231                parallel_group_entry(1, 2, 1, "activity"),
16232            ],
16233        ];
16234        let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
16235        let mut timer_fired = parallel_history_event(
16236            "TimerFired",
16237            3,
16238            "timer_id",
16239            "timer-3",
16240            deadline_path.clone(),
16241            None,
16242        );
16243        timer_fired.payload["delay_seconds"] = json!(0);
16244        timer_fired
16245            .raw
16246            .insert("id".to_string(), json!("timer-fired"));
16247        let mut timer_scheduled = parallel_history_event(
16248            "TimerScheduled",
16249            3,
16250            "timer_id",
16251            "timer-3",
16252            deadline_path,
16253            None,
16254        );
16255        timer_scheduled.payload["delay_seconds"] = json!(0);
16256        let history = vec![
16257            parallel_history_event(
16258                "ActivityScheduled",
16259                1,
16260                "activity_type",
16261                "nested-first",
16262                nested_paths[0].clone(),
16263                None,
16264            ),
16265            parallel_history_event(
16266                "ActivityScheduled",
16267                2,
16268                "activity_type",
16269                "nested-second",
16270                nested_paths[1].clone(),
16271                None,
16272            ),
16273            timer_scheduled,
16274            timer_fired,
16275            history_event(
16276                "SelectionResolved",
16277                json!({
16278                    "selection_group_id": "select-calls:1:3",
16279                    "selection_group_base_sequence": 1,
16280                    "selection_group_size": 3,
16281                    "member_key": "deadline",
16282                    "member_index": 1,
16283                    "member_base_sequence": 3,
16284                    "member_size": 1,
16285                    "operation_kind": "timer",
16286                    "operation_identity": "timer-3",
16287                    "outcome": "completed",
16288                    "resolution_event_id": "timer-fired",
16289                    "resolution_event_type": "TimerFired",
16290                }),
16291            ),
16292            parallel_history_event(
16293                "ActivityFailed",
16294                2,
16295                "activity_type",
16296                "nested-second",
16297                nested_paths[1].clone(),
16298                None,
16299            ),
16300        ];
16301        let ctx = workflow_context(history);
16302        let mut selection = Box::pin(ctx.select_keyed(vec![
16303            (
16304                "nested",
16305                ParallelOperation::group(vec![
16306                    ParallelOperation::activity("nested-first", json!([])),
16307                    ParallelOperation::activity("nested-second", json!([])),
16308                ]),
16309            ),
16310            ("deadline", ParallelOperation::timer(Duration::ZERO)),
16311        ]));
16312        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16313        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16314            panic!("deadline winner must replay");
16315        };
16316        let nested = selected
16317            .handle(&SelectionKey::Name("nested".to_string()))
16318            .expect("nested handle")
16319            .clone();
16320        let mut cancel = Box::pin(nested.cancel());
16321        assert!(matches!(
16322            cancel.as_mut().poll(&mut task_context),
16323            Poll::Ready(Ok(()))
16324        ));
16325        let mut await_nested = Box::pin(nested.await_result());
16326
16327        assert!(matches!(
16328            await_nested.as_mut().poll(&mut task_context),
16329            Poll::Ready(Err(Error::ActivityFailed(_)))
16330        ));
16331        assert!(ctx.take_commands().expect("commands").is_empty());
16332    }
16333
16334    #[test]
16335    fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16336        let ctx = workflow_context(Vec::new());
16337        let mut call = Box::pin(ctx.select(vec![
16338            ParallelOperation::child_workflow(
16339                "child",
16340                ChildWorkflowOptions::new("children"),
16341                json!([]),
16342            ),
16343            ParallelOperation::timer(Duration::from_secs(30)),
16344            ParallelOperation::signal("approval"),
16345            ParallelOperation::condition(
16346                ConditionWaitOptions::new("ready", "sha256:ready"),
16347                || Ok(false),
16348            ),
16349            ParallelOperation::group(vec![
16350                ParallelOperation::activity("nested-one", json!([])),
16351                ParallelOperation::activity("nested-two", json!([])),
16352            ]),
16353        ]));
16354        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16355        assert!(matches!(
16356            call.as_mut().poll(&mut task_context),
16357            Poll::Pending
16358        ));
16359        let commands = ctx.take_commands().expect("selection commands");
16360        assert_eq!(
16361            commands
16362                .iter()
16363                .map(|command| command["type"].as_str().unwrap_or_default())
16364                .collect::<Vec<_>>(),
16365            [
16366                "start_child_workflow",
16367                "start_timer",
16368                "open_signal_wait",
16369                "open_condition_wait",
16370                "schedule_activity",
16371                "schedule_activity",
16372            ]
16373        );
16374        assert!(commands.iter().all(|command| {
16375            command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16376        }));
16377        assert_eq!(
16378            commands[4]["parallel_group_path"].as_array().map(Vec::len),
16379            Some(2)
16380        );
16381        assert_eq!(
16382            commands[4]["parallel_group_path"][0]["selection_member_kind"],
16383            json!("group")
16384        );
16385        assert_eq!(
16386            commands[5]["parallel_group_path"][0]["selection_member_kind"],
16387            json!("group")
16388        );
16389
16390        let one_leaf_ctx = workflow_context(Vec::new());
16391        let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16392            ParallelOperation::activity("nested-only", json!([])),
16393        ])]));
16394        assert!(matches!(
16395            one_leaf.as_mut().poll(&mut task_context),
16396            Poll::Pending
16397        ));
16398        let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16399        assert_eq!(one_leaf_commands.len(), 1);
16400        assert_eq!(
16401            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16402            json!("group")
16403        );
16404        assert_eq!(
16405            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16406            json!(1)
16407        );
16408    }
16409
16410    async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16411        let mut saga = ctx.saga();
16412        let outcome = async {
16413            let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16414            saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16415            let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16416            saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16417            ctx.activity("trip.charge", json!([])).await?;
16418            Ok(json!({"status": "booked"}))
16419        }
16420        .await;
16421        saga.finish(outcome).await
16422    }
16423
16424    fn saga_activity(
16425        event_type: &str,
16426        sequence: u64,
16427        activity_type: &str,
16428        result: Option<Value>,
16429    ) -> HistoryEvent {
16430        let mut payload = json!({
16431            "sequence": sequence,
16432            "activity_type": activity_type,
16433            "message": format!("{activity_type} failed"),
16434            "exception_type": "PlannedFailure",
16435            "non_retryable": true,
16436        });
16437        if let Some(result) = result {
16438            payload["result"] = fixture_envelope(result);
16439        }
16440        history_event(event_type, payload)
16441    }
16442
16443    #[test]
16444    fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16445        let completed_hotel_compensation = saga_activity(
16446            "ActivityCompleted",
16447            4,
16448            "trip.cancel-hotel",
16449            Some(Value::Null),
16450        );
16451        let history = vec![
16452            saga_activity(
16453                "ActivityCompleted",
16454                1,
16455                "trip.reserve-flight",
16456                Some(json!("flight-1")),
16457            ),
16458            saga_activity(
16459                "ActivityCompleted",
16460                2,
16461                "trip.reserve-hotel",
16462                Some(json!("hotel-1")),
16463            ),
16464            saga_activity("ActivityFailed", 3, "trip.charge", None),
16465            completed_hotel_compensation.clone(),
16466            completed_hotel_compensation,
16467        ];
16468
16469        for _restart in 0..2 {
16470            let ctx = workflow_context(history.clone());
16471            let mut future = Box::pin(trip_saga(ctx.clone()));
16472            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16473            assert!(matches!(
16474                future.as_mut().poll(&mut task_context),
16475                Poll::Pending
16476            ));
16477            let commands = ctx.take_commands().expect("compensation command");
16478            assert_eq!(commands.len(), 1);
16479            assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16480        }
16481    }
16482
16483    #[test]
16484    fn saga_compensation_failure_preserves_both_typed_failures() {
16485        let history = vec![
16486            saga_activity(
16487                "ActivityCompleted",
16488                1,
16489                "trip.reserve-flight",
16490                Some(json!("flight-1")),
16491            ),
16492            saga_activity(
16493                "ActivityCompleted",
16494                2,
16495                "trip.reserve-hotel",
16496                Some(json!("hotel-1")),
16497            ),
16498            saga_activity("ActivityFailed", 3, "trip.charge", None),
16499            saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16500        ];
16501        let ctx = workflow_context(history);
16502        let mut future = Box::pin(trip_saga(ctx));
16503        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16504        let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16505            future.as_mut().poll(&mut task_context)
16506        else {
16507            panic!("compensation failure must remain structured");
16508        };
16509        assert!(matches!(
16510            *failure.initiating_failure,
16511            Error::ActivityFailed(_)
16512        ));
16513        assert!(matches!(
16514            *failure.compensation_failure,
16515            Error::ActivityFailed(_)
16516        ));
16517        assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16518        assert_eq!(failure.compensation_registration_order, 2);
16519    }
16520
16521    #[test]
16522    fn saga_compensates_cooperative_cancellation() {
16523        let ctx = workflow_context(vec![saga_activity(
16524            "ActivityCompleted",
16525            1,
16526            "trip.reserve-flight",
16527            Some(json!("flight-1")),
16528        )]);
16529        ctx.state.lock().expect("state").cancel_requested = true;
16530        let run = {
16531            let ctx = ctx.clone();
16532            async move {
16533                let mut saga = ctx.saga();
16534                let outcome = async {
16535                    let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16536                    saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16537                    ctx.throw_if_cancellation_requested()?;
16538                    Ok(json!("unexpected"))
16539                }
16540                .await;
16541                saga.finish(outcome).await
16542            }
16543        };
16544        let mut future = Box::pin(run);
16545        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16546        assert!(matches!(
16547            future.as_mut().poll(&mut task_context),
16548            Poll::Pending
16549        ));
16550        let commands = ctx.take_commands().expect("cancellation compensation");
16551        assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16552    }
16553
16554    fn workflow_task(
16555        workflow_type: &str,
16556        history_events: Vec<HistoryEvent>,
16557        payload_codec: &str,
16558    ) -> WorkflowTask {
16559        WorkflowTask {
16560            task_id: format!("wft-{workflow_type}"),
16561            workflow_command_id: None,
16562            workflow_id: Some(format!("wf-{workflow_type}")),
16563            run_id: Some(format!("run-{workflow_type}")),
16564            workflow_type: workflow_type.to_string(),
16565            cancel_requested: false,
16566            payload_codec: payload_codec.to_string(),
16567            arguments: Some(
16568                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16569            ),
16570            total_history_events: Some(history_events.len() as u64),
16571            history_size_bytes: None,
16572            continue_as_new_recommended: None,
16573            history_budget_pressure: None,
16574            history_events,
16575            next_history_page_token: None,
16576            workflow_task_attempt: 1,
16577            workflow_signal_id: None,
16578            signal_name: None,
16579            signal_arguments: None,
16580            workflow_update_id: None,
16581            update_name: None,
16582            lease_owner: Some("rust-worker".to_string()),
16583        }
16584    }
16585
16586    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16587    struct SideEffectProbe {
16588        request_id: String,
16589        attempt: u32,
16590    }
16591
16592    #[test]
16593    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16594        let calls = AtomicUsize::new(0);
16595        let ctx = workflow_context(Vec::new());
16596        let value = ctx
16597            .side_effect(|| {
16598                calls.fetch_add(1, Ordering::SeqCst);
16599                SideEffectProbe {
16600                    request_id: "request-42".to_string(),
16601                    attempt: 3,
16602                }
16603            })
16604            .expect("first side effect");
16605        assert_eq!(value.attempt, 3);
16606        assert_eq!(calls.load(Ordering::SeqCst), 1);
16607        let commands = ctx.take_commands().expect("commands");
16608        assert_eq!(commands.len(), 1);
16609        assert_eq!(commands[0]["type"], "record_side_effect");
16610        assert_eq!(
16611            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16612            serde_json::to_value(&value).expect("value")
16613        );
16614
16615        let replay = workflow_context(vec![history_event(
16616            "SideEffectRecorded",
16617            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16618        )]);
16619        let replayed: SideEffectProbe = replay
16620            .side_effect(|| {
16621                calls.fetch_add(1, Ordering::SeqCst);
16622                panic!("committed side-effect callbacks must not run during replay")
16623            })
16624            .expect("replayed side effect");
16625        assert_eq!(replayed, value);
16626        assert_eq!(calls.load(Ordering::SeqCst), 1);
16627        assert!(replay.take_commands().expect("commands").is_empty());
16628        replay.ensure_history_consumed().expect("history consumed");
16629    }
16630
16631    #[test]
16632    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16633        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16634        let value = ctx
16635            .side_effect(|| SideEffectProbe {
16636                request_id: "avro-request".to_string(),
16637                attempt: 1,
16638            })
16639            .expect("Avro side effect");
16640        let uuid = ctx.uuid_v4().expect("deterministic UUID");
16641        let commands = ctx.take_commands().expect("commands");
16642        assert_eq!(commands.len(), 2);
16643        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16644        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16645        assert_eq!(
16646            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16647            serde_json::to_value(&value).expect("value")
16648        );
16649
16650        let replay = workflow_context_with_codec(
16651            vec![
16652                history_event(
16653                    "SideEffectRecorded",
16654                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16655                ),
16656                history_event(
16657                    "SideEffectRecorded",
16658                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16659                ),
16660            ],
16661            DEFAULT_CODEC,
16662        );
16663        let replayed: SideEffectProbe = replay
16664            .side_effect(|| panic!("Avro callback must not run"))
16665            .expect("replayed Avro value");
16666        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16667        assert_eq!(replayed, value);
16668        assert_eq!(replayed_uuid, uuid);
16669        assert!(replay.take_commands().expect("commands").is_empty());
16670    }
16671
16672    #[test]
16673    fn typed_side_effect_replay_preserves_bytes_and_maps() {
16674        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16675        let value = ctx
16676            .side_effect_avro_value(typed_fidelity_probe)
16677            .expect("typed side effect");
16678        let commands = ctx.take_commands().expect("side-effect command");
16679        assert_eq!(
16680            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16681                .expect("recorded side effect"),
16682            value
16683        );
16684
16685        let replay = workflow_context_with_codec(
16686            vec![history_event(
16687                "SideEffectRecorded",
16688                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16689            )],
16690            DEFAULT_CODEC,
16691        );
16692        assert_eq!(
16693            replay
16694                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16695                .expect("replayed typed side effect"),
16696            value
16697        );
16698    }
16699
16700    #[test]
16701    fn ordered_side_effects_share_the_durable_command_stream() {
16702        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16703        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16704        let ctx = workflow_context(vec![
16705            history_event(
16706                "SideEffectRecorded",
16707                json!({"sequence": 1, "result": first}),
16708            ),
16709            history_event(
16710                "SideEffectRecorded",
16711                json!({"sequence": 2, "result": second}),
16712            ),
16713        ]);
16714        let first: String = ctx
16715            .side_effect(|| panic!("first callback must not run"))
16716            .expect("first replay");
16717        let second: i32 = ctx
16718            .side_effect(|| panic!("second callback must not run"))
16719            .expect("second replay");
16720        assert_eq!(first, "first");
16721        assert_eq!(second, 29);
16722        ctx.ensure_history_consumed().expect("ordered history");
16723
16724        let reordered = workflow_context(vec![history_event(
16725            "VersionMarkerRecorded",
16726            json!({
16727                "sequence": 1,
16728                "change_id": "before-side-effect",
16729                "version": 1,
16730                "min_supported": 1,
16731                "max_supported": 1,
16732            }),
16733        )]);
16734        let error = reordered
16735            .side_effect(|| "new".to_string())
16736            .expect_err("command reordering must fail");
16737        assert!(matches!(
16738            error,
16739            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16740                if reason == "recorded_command_mismatch"
16741        ));
16742    }
16743
16744    #[test]
16745    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16746        let ctx = workflow_context(Vec::new());
16747        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16748        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16749        assert!(ctx.patched("new-search").expect("patch"));
16750        ctx.deprecate_patch("new-search").expect("deprecate patch");
16751        let commands = ctx.take_commands().expect("commands");
16752        assert_eq!(commands.len(), 2);
16753        assert_eq!(commands[0]["type"], "record_version_marker");
16754        assert_eq!(commands[0]["version"], 2);
16755        assert_eq!(commands[1]["change_id"], "new-search");
16756
16757        let replay = workflow_context(vec![history_event(
16758            "VersionMarkerRecorded",
16759            json!({
16760                "sequence": 1,
16761                "change_id": "checkout-v2",
16762                "version": 2,
16763                "min_supported": 1,
16764                "max_supported": 2,
16765            }),
16766        )]);
16767        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16768        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16769        assert!(replay.take_commands().expect("commands").is_empty());
16770        replay.ensure_history_consumed().expect("history consumed");
16771    }
16772
16773    #[test]
16774    fn version_markers_reject_incompatible_or_malformed_history() {
16775        let incompatible = workflow_context(vec![history_event(
16776            "VersionMarkerRecorded",
16777            json!({
16778                "sequence": 1,
16779                "change_id": "checkout-v2",
16780                "version": 1,
16781                "min_supported": 1,
16782                "max_supported": 2,
16783            }),
16784        )]);
16785        let error = incompatible
16786            .get_version("checkout-v2", 2, 3)
16787            .expect_err("old version is unsupported");
16788        assert!(matches!(
16789            error,
16790            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16791                if reason == "version_marker_incompatible_range"
16792        ));
16793
16794        for (history, reason) in [
16795            (
16796                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16797                "side_effect_result_missing",
16798            ),
16799            (
16800                vec![history_event(
16801                    "SideEffectRecorded",
16802                    json!({
16803                        "sequence": 1,
16804                        "result": {"codec": "avro", "blob": "not-base64"},
16805                    }),
16806                )],
16807                "side_effect_payload_incompatible",
16808            ),
16809            (
16810                vec![history_event(
16811                    "SideEffectRecorded",
16812                    json!({"sequence": 1, "result": {"unwrapped": true}}),
16813                )],
16814                "side_effect_payload_malformed",
16815            ),
16816            (
16817                vec![history_event(
16818                    "VersionMarkerRecorded",
16819                    json!({
16820                        "sequence": 1,
16821                        "change_id": "change",
16822                        "version": 1,
16823                        "min_supported": 2,
16824                        "max_supported": 1,
16825                    }),
16826                )],
16827                "version_marker_history_range_invalid",
16828            ),
16829        ] {
16830            let error = WorkflowState::new(
16831                history,
16832                "rust-workers".to_string(),
16833                DEFAULT_CODEC.to_string(),
16834                None,
16835            )
16836            .expect_err("malformed history must fail");
16837            assert!(matches!(
16838                error,
16839                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16840                    if actual == reason
16841            ));
16842        }
16843    }
16844
16845    #[test]
16846    fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16847        let history = vec![history_event(
16848            "SearchAttributesUpserted",
16849            json!({
16850                "sequence": 1,
16851                "attributes": {"customer_tier": "gold"},
16852                "attribute_types": {"customer_tier": "keyword"},
16853                "merged": {"customer_tier": "gold"}
16854            }),
16855        )];
16856
16857        let matching = workflow_context(history.clone());
16858        matching
16859            .upsert_search_attributes(
16860                SearchAttributeUpdate::new()
16861                    .keyword("customer_tier", "gold")
16862                    .expect("keyword update"),
16863            )
16864            .expect("matching typed update must replay");
16865        matching
16866            .ensure_history_consumed()
16867            .expect("history consumed");
16868
16869        let changed_type = workflow_context(history.clone());
16870        let error = changed_type
16871            .upsert_search_attributes(
16872                SearchAttributeUpdate::new()
16873                    .string("customer_tier", "gold")
16874                    .expect("string update"),
16875            )
16876            .expect_err("same JSON value with a different declaration must be nondeterministic");
16877        let Error::NonDeterministicReplay(failure) = error else {
16878            panic!("typed identity drift must be a replay failure");
16879        };
16880        assert_eq!(failure.reason, "search_attribute_type_mismatch");
16881        assert_eq!(failure.sequence, Some(1));
16882
16883        let changed_value = workflow_context(history);
16884        let error = changed_value
16885            .upsert_search_attributes(
16886                SearchAttributeUpdate::new()
16887                    .keyword("customer_tier", "platinum")
16888                    .expect("keyword update"),
16889            )
16890            .expect_err("changed values must be nondeterministic");
16891        let Error::NonDeterministicReplay(failure) = error else {
16892            panic!("value drift must be a replay failure");
16893        };
16894        assert_eq!(failure.reason, "search_attribute_value_mismatch");
16895    }
16896
16897    #[test]
16898    fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16899        let history = vec![history_event(
16900            "SearchAttributesUpserted",
16901            json!({
16902                "sequence": 1,
16903                "attributes": {"customer_tier": "gold"},
16904                "merged": {"customer_tier": "gold"}
16905            }),
16906        )];
16907
16908        for update in [
16909            SearchAttributeUpdate::new()
16910                .keyword("customer_tier", "gold")
16911                .expect("keyword update"),
16912            SearchAttributeUpdate::new()
16913                .string("customer_tier", "gold")
16914                .expect("string update"),
16915        ] {
16916            let restarted = workflow_context(history.clone());
16917            restarted
16918                .upsert_search_attributes(update)
16919                .expect("legacy history constrains values but has unknown type identity");
16920            restarted
16921                .ensure_history_consumed()
16922                .expect("history consumed");
16923        }
16924    }
16925
16926    #[test]
16927    fn search_attribute_command_emits_canonical_types() {
16928        let ctx = workflow_context(Vec::new());
16929        ctx.upsert_search_attributes(
16930            SearchAttributeUpdate::new()
16931                .keyword("customer_tier", "gold")
16932                .expect("keyword update")
16933                .int("attempts", 3)
16934                .expect("int update")
16935                .delete("obsolete")
16936                .expect("delete update"),
16937        )
16938        .expect("valid search attributes");
16939
16940        assert_eq!(
16941            ctx.take_commands().expect("commands"),
16942            vec![json!({
16943                "type": "upsert_search_attributes",
16944                "attributes": {
16945                    "attempts": 3,
16946                    "customer_tier": "gold",
16947                    "obsolete": null
16948                },
16949                "attribute_types": {
16950                    "attempts": "int",
16951                    "customer_tier": "keyword"
16952                }
16953            })]
16954        );
16955    }
16956
16957    #[test]
16958    fn duplicate_side_effects_and_version_markers_are_rejected() {
16959        let duplicate_side_effect = WorkflowState::new(
16960            vec![
16961                history_event(
16962                    "SideEffectRecorded",
16963                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16964                ),
16965                history_event(
16966                    "SideEffectRecorded",
16967                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16968                ),
16969            ],
16970            "rust-workers".to_string(),
16971            DEFAULT_CODEC.to_string(),
16972            None,
16973        )
16974        .expect_err("duplicate side effect");
16975        assert!(matches!(
16976            duplicate_side_effect,
16977            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16978                if reason == "duplicate_side_effect_record"
16979        ));
16980
16981        let marker = |sequence| {
16982            history_event(
16983                "VersionMarkerRecorded",
16984                json!({
16985                    "sequence": sequence,
16986                    "change_id": "same-change",
16987                    "version": 1,
16988                    "min_supported": 1,
16989                    "max_supported": 1,
16990                }),
16991            )
16992        };
16993        let duplicate_marker = WorkflowState::new(
16994            vec![marker(1), marker(3)],
16995            "rust-workers".to_string(),
16996            DEFAULT_CODEC.to_string(),
16997            None,
16998        )
16999        .expect_err("duplicate marker");
17000        assert!(matches!(
17001            duplicate_marker,
17002            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17003                if reason == "duplicate_version_marker"
17004        ));
17005    }
17006
17007    #[test]
17008    fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
17009        let mut state = WorkflowState::new(
17010            Vec::new(),
17011            "rust-workers".to_string(),
17012            DEFAULT_CODEC.to_string(),
17013            None,
17014        )
17015        .expect("workflow state");
17016        state.workflow_command_identity = "command-7".to_string();
17017        let context = WorkflowContext {
17018            state: Arc::new(Mutex::new(state)),
17019        };
17020        let item =
17021            WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
17022
17023        context
17024            .append_workflow_stream("output", &[item], Some(10))
17025            .expect("append command");
17026        context
17027            .error_workflow_stream("output", "producer failed", None)
17028            .expect("error command");
17029        let commands = context.take_commands().expect("commands");
17030
17031        assert_eq!(commands[0]["type"], "record_side_effect");
17032        assert_eq!(
17033            commands[0]["workflow_stream"]["command_identity"],
17034            "command-7"
17035        );
17036        assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
17037        assert_eq!(
17038            commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
17039            "dw-stream:command-7:0:0"
17040        );
17041        assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
17042
17043        let recorded = history_event(
17044            "SideEffectRecorded",
17045            json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
17046        );
17047        let mut replay_state = WorkflowState::new(
17048            vec![recorded],
17049            "rust-workers".to_string(),
17050            DEFAULT_CODEC.to_string(),
17051            None,
17052        )
17053        .expect("replay state");
17054        replay_state.workflow_command_identity = "command-7".to_string();
17055        let replay_context = WorkflowContext {
17056            state: Arc::new(Mutex::new(replay_state)),
17057        };
17058        replay_context
17059            .append_workflow_stream(
17060                "output",
17061                &[WorkflowStreamAppendItem::from_reference(
17062                    "s3://bucket/item.avro",
17063                )],
17064                Some(10),
17065            )
17066            .expect("replayed append");
17067        assert!(replay_context
17068            .take_commands()
17069            .expect("replayed commands")
17070            .is_empty());
17071    }
17072
17073    #[test]
17074    fn workflow_stream_authoring_requires_server_durable_command_identity() {
17075        let context = workflow_context(Vec::new());
17076        let error = context
17077            .append_workflow_stream(
17078                "output",
17079                &[WorkflowStreamAppendItem::from_reference(
17080                    "s3://bucket/item.avro",
17081                )],
17082                None,
17083            )
17084            .expect_err("stream append without durable command identity must fail closed");
17085
17086        assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
17087        assert!(context.take_commands().expect("commands").is_empty());
17088    }
17089
17090    #[test]
17091    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
17092        fn worker(calls: Arc<AtomicUsize>) -> Worker {
17093            let client = Client::new("http://127.0.0.1:8080").expect("client");
17094            let mut worker = Worker::new(client, "rust-workers");
17095            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
17096                let calls = Arc::clone(&calls);
17097                async move {
17098                    let captured = ctx.side_effect(|| {
17099                        calls.fetch_add(1, Ordering::SeqCst);
17100                        "captured-once".to_string()
17101                    })?;
17102                    let version = ctx.get_version("cold-restart", 1, 2)?;
17103                    Ok(json!({"captured": captured, "version": version}))
17104                }
17105            });
17106            worker
17107        }
17108
17109        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
17110            WorkflowTask {
17111                task_id: "wft-side-effect-version".to_string(),
17112                workflow_command_id: None,
17113                workflow_id: Some("wf-side-effect-version".to_string()),
17114                run_id: Some("run-side-effect-version".to_string()),
17115                workflow_type: "rust.side-effect-version".to_string(),
17116                cancel_requested: false,
17117                payload_codec: DEFAULT_CODEC.to_string(),
17118                arguments: Some(
17119                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
17120                ),
17121                history_events,
17122                total_history_events: None,
17123                history_size_bytes: None,
17124                continue_as_new_recommended: None,
17125                history_budget_pressure: None,
17126                next_history_page_token: None,
17127                workflow_task_attempt: 1,
17128                workflow_signal_id: None,
17129                signal_name: None,
17130                signal_arguments: None,
17131                workflow_update_id: None,
17132                update_name: None,
17133                lease_owner: Some("rust-worker".to_string()),
17134            }
17135        }
17136
17137        let calls = Arc::new(AtomicUsize::new(0));
17138        let initial = worker(Arc::clone(&calls))
17139            .execute_workflow_task(task(Vec::new()))
17140            .expect("initial execution");
17141        assert_eq!(
17142            initial
17143                .iter()
17144                .map(|command| &command["type"])
17145                .collect::<Vec<_>>(),
17146            vec![
17147                "record_side_effect",
17148                "record_version_marker",
17149                "complete_workflow"
17150            ]
17151        );
17152        assert_eq!(calls.load(Ordering::SeqCst), 1);
17153
17154        let restarted = worker(Arc::clone(&calls));
17155        let replayed = restarted
17156            .execute_workflow_task(task(vec![
17157                history_event(
17158                    "SideEffectRecorded",
17159                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
17160                ),
17161                history_event(
17162                    "VersionMarkerRecorded",
17163                    json!({
17164                        "sequence": 2,
17165                        "change_id": "cold-restart",
17166                        "version": 2,
17167                        "min_supported": 1,
17168                        "max_supported": 2,
17169                    }),
17170                ),
17171            ]))
17172            .expect("cold replay");
17173        assert_eq!(replayed.len(), 1);
17174        assert_eq!(replayed[0]["type"], "complete_workflow");
17175        assert_eq!(calls.load(Ordering::SeqCst), 1);
17176    }
17177
17178    #[test]
17179    fn side_effect_replay_rejects_changed_rust_value_type() {
17180        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
17181        let ctx = workflow_context(vec![history_event(
17182            "SideEffectRecorded",
17183            json!({"sequence": 1, "result": result}),
17184        )]);
17185        let error = ctx
17186            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
17187            .expect_err("changed type must fail replay");
17188        assert!(matches!(
17189            error,
17190            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17191                if reason == "side_effect_type_mismatch"
17192        ));
17193    }
17194
17195    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
17196        vec![
17197            history_event(
17198                "ActivityScheduled",
17199                json!({
17200                    "sequence": 1,
17201                    "activity_type": "flaky",
17202                    "activity_execution_id": "act-1",
17203                    "activity": {
17204                        "id": "act-1",
17205                        "sequence": 1,
17206                        "type": "flaky",
17207                        "queue": "critical-activities",
17208                        "execution_mode": null,
17209                        "retry_policy": {
17210                            "snapshot_version": 1,
17211                            "max_attempts": 3,
17212                            "backoff_seconds": [2, 4],
17213                            "start_to_close_timeout": 30,
17214                            "schedule_to_start_timeout": 5,
17215                            "schedule_to_close_timeout": 90,
17216                            "heartbeat_timeout": 10,
17217                            "non_retryable_error_types": ["PermanentError"]
17218                        }
17219                    }
17220                }),
17221            ),
17222            history_event(
17223                "ActivityStarted",
17224                json!({
17225                    "sequence": 1,
17226                    "activity_type": "flaky",
17227                    "activity_execution_id": "act-1",
17228                    "activity_attempt_id": "attempt-1",
17229                    "attempt_number": 1
17230                }),
17231            ),
17232            history_event(
17233                "ActivityRetryScheduled",
17234                json!({
17235                    "sequence": 1,
17236                    "activity_type": "flaky",
17237                    "activity_execution_id": "act-1",
17238                    "activity_attempt_id": "attempt-1",
17239                    "attempt_number": 1,
17240                    "retry_after_attempt": 1,
17241                    "retry_backoff_seconds": 2,
17242                    "failure_category": "activity",
17243                    "exception_type": "TransientError"
17244                }),
17245            ),
17246            history_event(
17247                "ActivityStarted",
17248                json!({
17249                    "sequence": 1,
17250                    "activity_type": "flaky",
17251                    "activity_execution_id": "act-1",
17252                    "activity_attempt_id": "attempt-2",
17253                    "attempt_number": 2
17254                }),
17255            ),
17256            history_event(
17257                "ActivityCompleted",
17258                json!({
17259                    "sequence": 1,
17260                    "activity_type": "flaky",
17261                    "activity_execution_id": "act-1",
17262                    "activity_attempt_id": "attempt-2",
17263                    "attempt_number": 2,
17264                    "payload_codec": DEFAULT_CODEC,
17265                    "result": fixture_envelope(json!({"status":"recovered"}))
17266                }),
17267            ),
17268        ]
17269    }
17270
17271    fn retry_activity_options() -> ActivityOptions {
17272        ActivityOptions::new()
17273            .task_queue("critical-activities")
17274            .retry_policy(
17275                ActivityRetryPolicy::new(3)
17276                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
17277                    .non_retryable_error_type("PermanentError"),
17278            )
17279            .start_to_close_timeout(Duration::from_secs(30))
17280            .schedule_to_start_timeout(Duration::from_secs(5))
17281            .schedule_to_close_timeout(Duration::from_secs(90))
17282            .heartbeat_timeout(Duration::from_secs(10))
17283    }
17284
17285    #[test]
17286    fn fixed_avro_value_round_trips_json_values() {
17287        let value = json!({"greeting": "hello", "count": 3, "ok": true});
17288        let envelope = PayloadEnvelope::avro(&value).expect("encode");
17289        assert_eq!(envelope.codec, DEFAULT_CODEC);
17290        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
17291    }
17292
17293    #[tokio::test]
17294    async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
17295        let client = Client::new("http://127.0.0.1:8080").expect("client");
17296        let mut worker = Worker::new(client, "rust-workers");
17297        worker.register_typed_workflow(
17298            "typed.contract.workflow",
17299            |_ctx, input: TypedContract| async move { Ok(input) },
17300        );
17301        worker.register_typed_activity(
17302            "typed.contract.activity",
17303            |_ctx, input: TypedContract| async move { Ok(input) },
17304        );
17305
17306        let expected = typed_contract();
17307        let arguments = AvroValue::Array(vec![
17308            AvroValue::from_serialize(&expected).expect("typed request")
17309        ]);
17310        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
17311        let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
17312        workflow.arguments = Some(envelope.clone());
17313        let commands = worker
17314            .execute_workflow_task(workflow)
17315            .expect("typed workflow task");
17316        let workflow_result: TypedContract =
17317            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17318                .expect("workflow result envelope")
17319                .deserialize()
17320                .expect("workflow result type");
17321        assert_eq!(workflow_result, expected);
17322
17323        let activity = ActivityTask {
17324            task_id: "typed-contract-activity".to_string(),
17325            activity_attempt_id: Some("typed-contract-attempt".to_string()),
17326            attempt_id: None,
17327            activity_type: "typed.contract.activity".to_string(),
17328            payload_codec: DEFAULT_CODEC.to_string(),
17329            arguments: Some(envelope),
17330            attempt_number: 1,
17331            lease_owner: Some("rust-worker".to_string()),
17332        };
17333        let activity_result: TypedContract = worker
17334            .execute_activity_task(activity)
17335            .await
17336            .expect("typed activity task")
17337            .deserialize()
17338            .expect("activity result type");
17339        assert_eq!(activity_result, expected);
17340    }
17341
17342    #[tokio::test]
17343    async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17344        let client = Client::new("http://127.0.0.1:8080").expect("client");
17345        let mut worker = Worker::new(client, "rust-workers");
17346        worker.register_typed_workflow(
17347            "typed.shape.workflow",
17348            |_ctx, input: TypedContract| async move { Ok(input) },
17349        );
17350        worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17351            Ok(f64::NAN)
17352        });
17353
17354        let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17355        workflow.arguments = Some(
17356            encode_typed_envelope(
17357                &AvroValue::Array(vec![
17358                    AvroValue::String("first".to_string()),
17359                    AvroValue::String("second".to_string()),
17360                ]),
17361                DEFAULT_CODEC,
17362            )
17363            .expect("malformed typed arguments"),
17364        );
17365        let commands = worker
17366            .execute_workflow_task(workflow)
17367            .expect("shape mismatch becomes a workflow failure");
17368        let message = commands[0]["message"].as_str().expect("failure message");
17369        assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17370        assert!(message.contains(type_name::<TypedContract>()));
17371        assert!(message.contains("task carried 2 arguments"));
17372
17373        let activity = ActivityTask {
17374            task_id: "typed-unsupported-activity".to_string(),
17375            activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17376            attempt_id: None,
17377            activity_type: "typed.unsupported.activity".to_string(),
17378            payload_codec: DEFAULT_CODEC.to_string(),
17379            arguments: Some(
17380                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17381                    .expect("unit arguments"),
17382            ),
17383            attempt_number: 1,
17384            lease_owner: Some("rust-worker".to_string()),
17385        };
17386        let Error::HandlerType {
17387            handler_kind,
17388            handler_name,
17389            value_kind,
17390            rust_type,
17391            message,
17392        } = worker
17393            .execute_activity_task(activity)
17394            .await
17395            .expect_err("non-finite handler output must fail")
17396        else {
17397            panic!("expected contextual handler type failure");
17398        };
17399        assert_eq!(handler_kind, HandlerKind::Activity);
17400        assert_eq!(handler_name, "typed.unsupported.activity");
17401        assert_eq!(value_kind, HandlerValueKind::Result);
17402        assert_eq!(rust_type, type_name::<f64>());
17403        assert!(message.contains("non_finite_float"));
17404    }
17405
17406    #[tokio::test]
17407    async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17408        #[derive(Clone, Default)]
17409        struct State {
17410            observed: Option<TypedContract>,
17411        }
17412
17413        let client = Client::new("http://127.0.0.1:8080").expect("client");
17414        let mut worker = Worker::new(client, "rust-workers");
17415        worker.register_typed_replayed_workflow(
17416            "typed.contract.replayed",
17417            State::default,
17418            |ctx, input: TypedContract, state| async move {
17419                let result: TypedContract =
17420                    ctx.activity_typed("typed.contract.activity", input).await?;
17421                state.update(|current| current.observed = Some(result.clone()))?;
17422                Ok(result)
17423            },
17424        );
17425        worker.register_replayed_query::<State, _, _>(
17426            "typed.contract.replayed",
17427            "observed",
17428            |_ctx, state, _args| async move {
17429                Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17430            },
17431        );
17432
17433        let expected = typed_contract();
17434        let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17435        let workflow_arguments =
17436            encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17437                .expect("workflow arguments");
17438        let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17439        let task = QueryTask {
17440            query_task_id: "typed-replay-query".to_string(),
17441            query_task_attempt: 1,
17442            lease_owner: Some("rust-worker".to_string()),
17443            workflow_id: Some("typed-replay".to_string()),
17444            run_id: Some("typed-replay-run".to_string()),
17445            workflow_type: "typed.contract.replayed".to_string(),
17446            query_name: "observed".to_string(),
17447            payload_codec: DEFAULT_CODEC.to_string(),
17448            workflow_arguments: Some(workflow_arguments),
17449            query_arguments: Some(
17450                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17451                    .expect("query arguments"),
17452            ),
17453            history_events: vec![
17454                history_event(
17455                    "ActivityScheduled",
17456                    json!({
17457                        "sequence": 1,
17458                        "activity_type": "typed.contract.activity"
17459                    }),
17460                ),
17461                history_event(
17462                    "ActivityCompleted",
17463                    json!({
17464                        "sequence": 1,
17465                        "activity_type": "typed.contract.activity",
17466                        "payload_codec": DEFAULT_CODEC,
17467                        "result": result
17468                    }),
17469                ),
17470            ],
17471            history_export: None,
17472            run_status: Some("completed".to_string()),
17473        };
17474
17475        assert_eq!(
17476            worker
17477                .execute_query_task(task)
17478                .await
17479                .expect("typed replay query")
17480                .deserialize::<i64>()
17481                .expect("query result"),
17482            expected.signed
17483        );
17484    }
17485
17486    #[tokio::test]
17487    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17488        let client = Client::new("http://127.0.0.1:8080").expect("client");
17489        let mut worker = Worker::new(client, "rust-workers");
17490        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17491        worker
17492            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17493        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17494            Ok(input)
17495        });
17496        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17497            Ok(input)
17498        });
17499        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17500            Ok(AvroValue::Array(
17501                ctx.wait_signal_avro_value("changed").await?,
17502            ))
17503        });
17504
17505        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17506        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17507
17508        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17509        workflow.arguments = Some(envelope.clone());
17510        let commands = worker
17511            .execute_workflow_task(workflow)
17512            .expect("typed workflow task");
17513        assert_eq!(commands[0]["type"], "complete_workflow");
17514        assert_eq!(
17515            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17516                .expect("typed workflow result"),
17517            arguments
17518        );
17519
17520        let activity = ActivityTask {
17521            task_id: "activity-typed".to_string(),
17522            activity_attempt_id: Some("attempt-typed".to_string()),
17523            attempt_id: None,
17524            activity_type: "typed.activity".to_string(),
17525            payload_codec: DEFAULT_CODEC.to_string(),
17526            arguments: Some(envelope.clone()),
17527            attempt_number: 1,
17528            lease_owner: Some("rust-worker".to_string()),
17529        };
17530        assert_eq!(
17531            worker
17532                .execute_activity_task(activity)
17533                .await
17534                .expect("typed activity result"),
17535            arguments
17536        );
17537
17538        let query = QueryTask {
17539            query_task_id: "query-typed".to_string(),
17540            query_task_attempt: 1,
17541            lease_owner: Some("rust-worker".to_string()),
17542            workflow_id: Some("typed-1".to_string()),
17543            run_id: Some("run-typed".to_string()),
17544            workflow_type: "typed.echo".to_string(),
17545            query_name: "inspect".to_string(),
17546            payload_codec: DEFAULT_CODEC.to_string(),
17547            workflow_arguments: Some(
17548                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17549                    .expect("workflow input"),
17550            ),
17551            query_arguments: Some(envelope.clone()),
17552            history_events: Vec::new(),
17553            history_export: None,
17554            run_status: Some("running".to_string()),
17555        };
17556        assert_eq!(
17557            worker
17558                .execute_query_task(query)
17559                .await
17560                .expect("typed query result"),
17561            arguments
17562        );
17563
17564        let mut update = workflow_task(
17565            "typed.echo",
17566            vec![history_event(
17567                "UpdateAccepted",
17568                json!({
17569                    "update_id": "update-typed",
17570                    "update_name": "replace",
17571                    "arguments": envelope.clone(),
17572                }),
17573            )],
17574            DEFAULT_CODEC,
17575        );
17576        update.workflow_update_id = Some("update-typed".to_string());
17577        update.update_name = Some("replace".to_string());
17578        let commands = worker
17579            .execute_workflow_task(update)
17580            .expect("typed update task");
17581        assert_eq!(commands[0]["type"], "complete_update");
17582        assert_eq!(
17583            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17584                .expect("typed update result"),
17585            arguments
17586        );
17587
17588        let mut signal = workflow_task(
17589            "typed.signal",
17590            vec![history_event(
17591                "SignalReceived",
17592                json!({
17593                    "signal_id": "signal-typed",
17594                    "signal_name": "changed",
17595                    "arguments": envelope.clone(),
17596                }),
17597            )],
17598            DEFAULT_CODEC,
17599        );
17600        signal.workflow_signal_id = Some("signal-typed".to_string());
17601        signal.signal_name = Some("changed".to_string());
17602        signal.signal_arguments = Some(envelope);
17603        let commands = worker
17604            .execute_workflow_task(signal)
17605            .expect("typed signal resume");
17606        assert_eq!(
17607            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17608                .expect("typed signal result"),
17609            arguments
17610        );
17611    }
17612
17613    #[tokio::test]
17614    async fn typed_helpers_never_parse_json_inspection_projection() {
17615        let collision_values = projection_collision_probe();
17616        let expected = AvroValue::Array(collision_values.clone());
17617        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17618
17619        let activity_context = workflow_context_with_codec(
17620            vec![history_event(
17621                "ActivityCompleted",
17622                json!({
17623                    "sequence": 1,
17624                    "activity_type": "collision.activity",
17625                    "payload_codec": DEFAULT_CODEC,
17626                    "result": envelope.clone(),
17627                }),
17628            )],
17629            DEFAULT_CODEC,
17630        );
17631        assert_eq!(
17632            activity_context
17633                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17634                .await
17635                .expect("typed activity collision result"),
17636            expected
17637        );
17638
17639        let signal_context = workflow_context_with_codec(
17640            vec![
17641                history_event(
17642                    "SignalWaitOpened",
17643                    json!({"sequence": 1, "signal_name": "collision"}),
17644                ),
17645                history_event(
17646                    "SignalApplied",
17647                    json!({
17648                        "sequence": 1,
17649                        "signal_name": "collision",
17650                        "payload_codec": DEFAULT_CODEC,
17651                        "value": envelope.clone(),
17652                    }),
17653                ),
17654            ],
17655            DEFAULT_CODEC,
17656        );
17657        assert_eq!(
17658            signal_context
17659                .wait_signal_avro_value("collision")
17660                .await
17661                .expect("typed signal collision arguments"),
17662            collision_values
17663        );
17664
17665        let child_context = workflow_context_with_codec(
17666            vec![
17667                history_event(
17668                    "ChildWorkflowScheduled",
17669                    json!({
17670                        "sequence": 1,
17671                        "child_workflow_instance_id": "collision-child",
17672                        "child_workflow_run_id": "collision-run",
17673                        "child_workflow_type": "collision.child",
17674                    }),
17675                ),
17676                history_event(
17677                    "ChildRunCompleted",
17678                    json!({
17679                        "sequence": 1,
17680                        "child_workflow_instance_id": "collision-child",
17681                        "child_workflow_run_id": "collision-run",
17682                        "child_workflow_type": "collision.child",
17683                        "payload_codec": DEFAULT_CODEC,
17684                        "result": envelope,
17685                    }),
17686                ),
17687            ],
17688            DEFAULT_CODEC,
17689        );
17690        let child = child_context
17691            .start_child_workflow_avro_value(
17692                "collision.child",
17693                ChildWorkflowOptions::new("collision-workers"),
17694                AvroValue::Array(Vec::new()),
17695            )
17696            .await
17697            .expect("typed child collision result");
17698        assert_eq!(child.result, expected);
17699    }
17700
17701    #[tokio::test]
17702    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17703        let client = Client::new("http://127.0.0.1:8080").expect("client");
17704        let mut worker = Worker::new(client, "rust-workers");
17705        worker.register_replayed_workflow_avro_value(
17706            "typed.replayed",
17707            || (),
17708            |_ctx, input, _state| async move { Ok(input) },
17709        );
17710        worker.register_replayed_query_avro_value::<(), _, _>(
17711            "typed.replayed",
17712            "inspect",
17713            |ctx, _state, args| async move {
17714                let mut signals = ctx.signals_avro_value("collision");
17715                let signal = signals
17716                    .pop()
17717                    .map(AvroValue::Array)
17718                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17719                Ok(AvroValue::Array(vec![
17720                    ctx.workflow_input_avro_value().clone(),
17721                    signal,
17722                    args,
17723                ]))
17724            },
17725        );
17726        let arguments = AvroValue::Array(projection_collision_probe());
17727        let signal_arguments =
17728            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17729        let task = QueryTask {
17730            query_task_id: "query-typed-replay".to_string(),
17731            query_task_attempt: 1,
17732            lease_owner: Some("rust-worker".to_string()),
17733            workflow_id: Some("typed-replay".to_string()),
17734            run_id: Some("run-typed-replay".to_string()),
17735            workflow_type: "typed.replayed".to_string(),
17736            query_name: "inspect".to_string(),
17737            payload_codec: DEFAULT_CODEC.to_string(),
17738            workflow_arguments: Some(
17739                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17740            ),
17741            query_arguments: Some(
17742                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17743            ),
17744            history_events: vec![history_event(
17745                "SignalReceived",
17746                json!({
17747                    "signal_id": "collision-signal",
17748                    "signal_name": "collision",
17749                    "workflow_sequence": 1,
17750                    "payload_codec": DEFAULT_CODEC,
17751                    "arguments": signal_arguments,
17752                }),
17753            )],
17754            history_export: None,
17755            run_status: Some("completed".to_string()),
17756        };
17757
17758        assert_eq!(
17759            worker
17760                .execute_query_task(task)
17761                .await
17762                .expect("typed replay query"),
17763            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17764        );
17765    }
17766
17767    #[test]
17768    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17769        let value = BTreeMap::from([(1_i32, "integer key")]);
17770        let error = PayloadEnvelope::avro(&value)
17771            .expect_err("integer map keys must fail")
17772            .to_string();
17773
17774        assert!(error.contains("invalid_map_key"));
17775    }
17776
17777    #[test]
17778    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17779        let envelope = PayloadEnvelope {
17780            codec: "json".to_string(),
17781            blob: r#"{"greeting":"hello"}"#.to_string(),
17782        };
17783
17784        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17785        let diagnostic = error.to_string();
17786        assert!(diagnostic.contains("unsupported_payload_codec"));
17787        assert!(diagnostic.contains("codec=\"avro\""));
17788        assert!(diagnostic.contains("HTTP document transport"));
17789    }
17790
17791    #[test]
17792    fn untagged_json_payload_value_fails_closed() {
17793        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17794            .expect_err("untagged JSON payload values must fail");
17795        let diagnostic = error.to_string();
17796        assert!(diagnostic.contains("unsupported_payload_codec"));
17797        assert!(diagnostic.contains("untagged durable payload"));
17798        assert!(diagnostic.contains("HTTP document transport"));
17799    }
17800
17801    #[test]
17802    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17803        let envelope = PayloadEnvelope {
17804            codec: DEFAULT_CODEC.to_string(),
17805            blob: BASE64.encode([0x01]),
17806        };
17807
17808        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17809        assert!(error.to_string().contains("invalid_payload_framing"));
17810    }
17811
17812    #[tokio::test]
17813    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17814        let server = MockWorkerServer::start();
17815        let client = Client::builder(server.base_url())
17816            .timeout(Duration::from_secs(2))
17817            .build()
17818            .expect("client");
17819        let invalid_commands = [
17820            json!({
17821                "type": "complete_workflow",
17822                "result": {"codec": "json", "blob": null}
17823            }),
17824            json!({
17825                "type": "schedule_activity",
17826                "arguments": {"codec": "yaml", "blob": "ignored"}
17827            }),
17828            json!({
17829                "type": "start_child_workflow",
17830                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17831            }),
17832            json!({"type": "continue_as_new", "arguments": []}),
17833            json!({"type": "complete_update"}),
17834            json!({"type": "record_side_effect", "result": null}),
17835            json!({
17836                "type": "start_service_operation",
17837                "payload_codec": DEFAULT_CODEC,
17838                "request_payload": "raw-avro-bytes"
17839            }),
17840        ];
17841
17842        for command in invalid_commands {
17843            let error = client
17844                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17845                .await
17846                .expect_err("invalid durable payload must fail locally");
17847            let diagnostic = error.to_string();
17848            assert!(
17849                diagnostic.contains("unsupported_payload_codec")
17850                    || diagnostic.contains("invalid_payload_envelope")
17851                    || diagnostic.contains("untagged durable payload"),
17852                "unexpected validation diagnostic: {diagnostic}"
17853            );
17854        }
17855
17856        assert_eq!(
17857            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17858            0,
17859            "invalid command payloads must not reach HTTP transport"
17860        );
17861    }
17862
17863    #[test]
17864    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17865        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17866        let commands = [
17867            json!({"type": "complete_workflow", "result": envelope.clone()}),
17868            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17869            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17870            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17871            json!({"type": "complete_update", "result": envelope.clone()}),
17872            json!({"type": "record_side_effect", "result": envelope.clone()}),
17873            json!({
17874                "type": "start_service_operation",
17875                "payload_codec": DEFAULT_CODEC,
17876                "request_payload": envelope.clone()
17877            }),
17878            json!({
17879                "type": "complete_workflow",
17880                "result": envelope,
17881                "metadata": {
17882                    "codec": "json",
17883                    "payload_codec": "customer-codec",
17884                    "result": {"codec": "yaml", "blob": null}
17885                }
17886            }),
17887        ];
17888
17889        validate_workflow_task_commands(&commands)
17890            .expect("customer metadata must not become a protocol codec declaration");
17891    }
17892
17893    #[test]
17894    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17895        assert_eq!(
17896            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17897            AvroValue::Array(Vec::new())
17898        );
17899        assert_eq!(
17900            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17901            AvroValue::Array(Vec::new())
17902        );
17903
17904        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17905        signal.signal_name = Some("empty-signal".to_string());
17906        signal.signal_arguments = None;
17907        let decoded = decode_resume_signal(&signal)
17908            .expect("valid Avro signal")
17909            .expect("named signal resumes the workflow");
17910        assert!(decoded.arguments.is_empty());
17911    }
17912
17913    #[tokio::test]
17914    async fn malformed_task_level_codecs_become_pre_handler_failures() {
17915        let client = Client::new("http://127.0.0.1:8080").expect("client");
17916        let mut worker = Worker::new(client, "rust-workers");
17917        let handler_calls = Arc::new(AtomicUsize::new(0));
17918
17919        let calls = Arc::clone(&handler_calls);
17920        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17921            calls.fetch_add(1, Ordering::SeqCst);
17922            async move { Ok(Value::Null) }
17923        });
17924        let calls = Arc::clone(&handler_calls);
17925        worker.register_activity("codec.activity", move |_ctx, _args| {
17926            calls.fetch_add(1, Ordering::SeqCst);
17927            async move { Ok(Value::Null) }
17928        });
17929        let calls = Arc::clone(&handler_calls);
17930        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17931            calls.fetch_add(1, Ordering::SeqCst);
17932            async move { Ok(Value::Null) }
17933        });
17934
17935        let mut failures = Vec::new();
17936        for codec_case in [
17937            InvalidTaskPayloadCodec::Missing,
17938            InvalidTaskPayloadCodec::Null,
17939            InvalidTaskPayloadCodec::NonString,
17940        ] {
17941            let mut workflow = json!({
17942                "task_id": format!("workflow-{}", codec_case.label()),
17943                "workflow_type": "codec.workflow"
17944            });
17945            codec_case.apply(&mut workflow);
17946            match serde_json::from_value::<WorkflowTask>(workflow) {
17947                Ok(task) => match worker.execute_workflow_task(task) {
17948                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17949                    outcome => failures.push(format!(
17950                        "workflow {} codec returned {outcome:?}",
17951                        codec_case.label()
17952                    )),
17953                },
17954                Err(error) => failures.push(format!(
17955                    "workflow {} codec failed transport deserialization: {error}",
17956                    codec_case.label()
17957                )),
17958            }
17959
17960            let mut activity = json!({
17961                "task_id": format!("activity-{}", codec_case.label()),
17962                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17963                "activity_type": "codec.activity",
17964                "attempt_number": 1
17965            });
17966            codec_case.apply(&mut activity);
17967            match serde_json::from_value::<ActivityTask>(activity) {
17968                Ok(task) => match worker.execute_activity_task(task).await {
17969                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17970                    outcome => failures.push(format!(
17971                        "activity {} codec returned {outcome:?}",
17972                        codec_case.label()
17973                    )),
17974                },
17975                Err(error) => failures.push(format!(
17976                    "activity {} codec failed transport deserialization: {error}",
17977                    codec_case.label()
17978                )),
17979            }
17980
17981            let mut query = json!({
17982                "query_task_id": format!("query-{}", codec_case.label()),
17983                "workflow_type": "codec.workflow",
17984                "query_name": "known"
17985            });
17986            codec_case.apply(&mut query);
17987            match serde_json::from_value::<QueryTask>(query) {
17988                Ok(task) => match worker.execute_query_task(task).await {
17989                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17990                    outcome => failures.push(format!(
17991                        "query {} codec returned {outcome:?}",
17992                        codec_case.label()
17993                    )),
17994                },
17995                Err(error) => failures.push(format!(
17996                    "query {} codec failed transport deserialization: {error}",
17997                    codec_case.label()
17998                )),
17999            }
18000        }
18001
18002        assert!(failures.is_empty(), "{}", failures.join("\n"));
18003        assert_eq!(
18004            handler_calls.load(Ordering::SeqCst),
18005            0,
18006            "invalid task codecs must not invoke a handler"
18007        );
18008    }
18009
18010    #[tokio::test]
18011    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
18012        for codec_case in [
18013            InvalidTaskPayloadCodec::Missing,
18014            InvalidTaskPayloadCodec::Null,
18015            InvalidTaskPayloadCodec::NonString,
18016        ] {
18017            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
18018            let client = Client::builder(server.base_url())
18019                .timeout(Duration::from_secs(2))
18020                .build()
18021                .expect("client");
18022            let mut worker = Worker::new(client, "rust-workers")
18023                .worker_id("codec-worker")
18024                .poll_timeout(Duration::from_millis(10));
18025            let handler_calls = Arc::new(AtomicUsize::new(0));
18026
18027            let calls = Arc::clone(&handler_calls);
18028            worker.register_workflow("codec.workflow", move |_ctx, _args| {
18029                calls.fetch_add(1, Ordering::SeqCst);
18030                async move { Ok(Value::Null) }
18031            });
18032            let calls = Arc::clone(&handler_calls);
18033            worker.register_activity("codec.activity", move |_ctx, _args| {
18034                calls.fetch_add(1, Ordering::SeqCst);
18035                async move { Ok(Value::Null) }
18036            });
18037            let calls = Arc::clone(&handler_calls);
18038            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
18039                calls.fetch_add(1, Ordering::SeqCst);
18040                async move { Ok(Value::Null) }
18041            });
18042
18043            assert_eq!(
18044                worker.run_once().await.expect("invalid tasks are settled"),
18045                3,
18046                "all {} codec tasks must be handled",
18047                codec_case.label()
18048            );
18049            assert_eq!(
18050                handler_calls.load(Ordering::SeqCst),
18051                0,
18052                "{} task codecs must fail before every handler",
18053                codec_case.label()
18054            );
18055
18056            for path in [
18057                "/api/worker/workflow-tasks/codec-workflow/fail",
18058                "/api/worker/activity-tasks/codec-activity/fail",
18059                "/api/worker/query-tasks/codec-query/fail",
18060            ] {
18061                let body = server.request_body(path);
18062                assert!(
18063                    body["failure"]["message"]
18064                        .as_str()
18065                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
18066                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
18067                    codec_case.label()
18068                );
18069            }
18070            assert_eq!(
18071                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
18072                    ["reason"],
18073                "query_payload_decode_failed"
18074            );
18075            for path in [
18076                "/api/worker/workflow-tasks/codec-workflow/complete",
18077                "/api/worker/activity-tasks/codec-activity/complete",
18078                "/api/worker/query-tasks/codec-query/complete",
18079            ] {
18080                assert_eq!(
18081                    server.request_count(path),
18082                    0,
18083                    "invalid {} codec task reached {path}",
18084                    codec_case.label()
18085                );
18086            }
18087        }
18088    }
18089
18090    #[tokio::test]
18091    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
18092        let client = Client::new("http://127.0.0.1:8080").expect("client");
18093        let mut worker = Worker::new(client, "rust-workers");
18094        let handler_calls = Arc::new(AtomicUsize::new(0));
18095
18096        let calls = Arc::clone(&handler_calls);
18097        worker.register_workflow("codec.workflow", move |_ctx, _args| {
18098            calls.fetch_add(1, Ordering::SeqCst);
18099            async move { Ok(Value::Null) }
18100        });
18101        let calls = Arc::clone(&handler_calls);
18102        worker.register_activity("codec.activity", move |_ctx, _args| {
18103            calls.fetch_add(1, Ordering::SeqCst);
18104            async move { Ok(Value::Null) }
18105        });
18106        let calls = Arc::clone(&handler_calls);
18107        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
18108            calls.fetch_add(1, Ordering::SeqCst);
18109            async move { Ok(Value::Null) }
18110        });
18111        let calls = Arc::clone(&handler_calls);
18112        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
18113            calls.fetch_add(1, Ordering::SeqCst);
18114            async move { Ok(Value::Null) }
18115        });
18116
18117        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18118        workflow.payload_codec = "json".to_string();
18119        workflow.arguments = None;
18120        let error = worker
18121            .execute_workflow_task(workflow)
18122            .expect_err("task codec must be checked before workflow invocation");
18123        assert!(error.to_string().contains("unsupported_payload_codec"));
18124
18125        let activity = ActivityTask {
18126            task_id: "activity-invalid-codec".to_string(),
18127            activity_attempt_id: None,
18128            attempt_id: None,
18129            activity_type: "codec.activity".to_string(),
18130            payload_codec: "unknown".to_string(),
18131            arguments: None,
18132            attempt_number: 1,
18133            lease_owner: None,
18134        };
18135        let error = worker
18136            .execute_activity_task(activity)
18137            .await
18138            .expect_err("task codec must be checked before activity invocation");
18139        assert!(error.to_string().contains("unsupported_payload_codec"));
18140
18141        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18142        update.workflow_update_id = Some("update-invalid-codec".to_string());
18143        update.update_name = Some("known".to_string());
18144        update.history_events.push(history_event(
18145            "UpdateAccepted",
18146            json!({
18147                "update_id": "update-invalid-codec",
18148                "update_name": "known",
18149                "arguments": {"codec": "json", "blob": null}
18150            }),
18151        ));
18152        let error = worker
18153            .execute_workflow_task(update)
18154            .expect_err("nested update codec must be checked before handler lookup");
18155        assert!(error.to_string().contains("unsupported_payload_codec"));
18156
18157        let query: QueryTask = serde_json::from_value(json!({
18158            "query_task_id": "query-invalid-codec",
18159            "workflow_type": "codec.workflow",
18160            "query_name": "known",
18161            "payload_codec": DEFAULT_CODEC,
18162            "workflow_arguments": null,
18163            "query_arguments": null,
18164            "history_export": {
18165                "payloads": {"codec": DEFAULT_CODEC},
18166                "signals": [{
18167                    "name": "empty",
18168                    "payload_codec": "json",
18169                    "arguments": null
18170                }]
18171            }
18172        }))
18173        .expect("query task");
18174        let failure = worker
18175            .execute_query_task(query)
18176            .await
18177            .expect_err("exported signal codec must be checked before query invocation");
18178        assert_eq!(failure.reason, "query_payload_decode_failed");
18179        assert!(failure.message.contains("unsupported_payload_codec"));
18180
18181        let exported_history: QueryTask = serde_json::from_value(json!({
18182            "query_task_id": "query-invalid-history-codec",
18183            "workflow_type": "codec.workflow",
18184            "query_name": "known",
18185            "payload_codec": DEFAULT_CODEC,
18186            "history_export": {
18187                "payloads": {"codec": DEFAULT_CODEC},
18188                "history_events": [{
18189                    "type": "ActivityCompleted",
18190                    "payload": {"payload_codec": "unknown", "result": null}
18191                }]
18192            }
18193        }))
18194        .expect("query task");
18195        let failure = worker
18196            .execute_query_task(exported_history)
18197            .await
18198            .expect_err("exported history codec must be checked before query invocation");
18199        assert_eq!(failure.reason, "query_payload_decode_failed");
18200        assert!(failure.message.contains("unsupported_payload_codec"));
18201        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
18202
18203        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
18204        unknown_workflow.arguments = None;
18205        unknown_workflow.history_events.push(history_event(
18206            "SignalReceived",
18207            json!({
18208                "signal_name": "empty",
18209                "payload_codec": "json",
18210                "arguments": null
18211            }),
18212        ));
18213        let error = worker
18214            .execute_workflow_task(unknown_workflow)
18215            .expect_err("history codec must precede unknown workflow outcome");
18216        assert!(error.to_string().contains("unsupported_payload_codec"));
18217
18218        let unknown_activity = ActivityTask {
18219            task_id: "activity-unknown".to_string(),
18220            activity_attempt_id: None,
18221            attempt_id: None,
18222            activity_type: "missing".to_string(),
18223            payload_codec: "json".to_string(),
18224            arguments: None,
18225            attempt_number: 1,
18226            lease_owner: None,
18227        };
18228        let error = worker
18229            .execute_activity_task(unknown_activity)
18230            .await
18231            .expect_err("codec must precede unknown activity outcome");
18232        assert!(error.to_string().contains("unsupported_payload_codec"));
18233
18234        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18235        unknown_update.payload_codec = "json".to_string();
18236        unknown_update.arguments = None;
18237        unknown_update.workflow_update_id = Some("update-unknown".to_string());
18238        unknown_update.update_name = Some("missing".to_string());
18239        let error = worker
18240            .execute_workflow_task(unknown_update)
18241            .expect_err("codec must precede fail_update shortcut");
18242        assert!(error.to_string().contains("unsupported_payload_codec"));
18243
18244        let unknown_query: QueryTask = serde_json::from_value(json!({
18245            "query_task_id": "query-unknown",
18246            "workflow_type": "missing",
18247            "query_name": "missing",
18248            "payload_codec": "json",
18249            "workflow_arguments": null,
18250            "query_arguments": null
18251        }))
18252        .expect("query task");
18253        let failure = worker
18254            .execute_query_task(unknown_query)
18255            .await
18256            .expect_err("codec must precede unknown query outcome");
18257        assert_eq!(failure.reason, "query_payload_decode_failed");
18258        assert!(failure.message.contains("unsupported_payload_codec"));
18259    }
18260
18261    #[tokio::test]
18262    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
18263        let client = Client::new("http://127.0.0.1:8080").expect("client");
18264        let worker = Worker::new(client, "rust-workers");
18265
18266        for event_type in ["SignalReceived", "SignalApplied"] {
18267            for (payload_field, codec) in [
18268                ("value", "json"),
18269                ("input", "unknown"),
18270                ("arguments", "json"),
18271            ] {
18272                let payload = json!({
18273                    "signal_name": "empty",
18274                    payload_field: {"codec": codec, "blob": null}
18275                });
18276                let workflow = workflow_task(
18277                    "missing",
18278                    vec![history_event(event_type, payload.clone())],
18279                    DEFAULT_CODEC,
18280                );
18281                let error = worker
18282                    .execute_workflow_task(workflow)
18283                    .expect_err("signal payload codec must precede unknown workflow outcome");
18284                assert!(
18285                    error.to_string().contains("unsupported_payload_codec"),
18286                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
18287                );
18288
18289                let query: QueryTask = serde_json::from_value(json!({
18290                    "query_task_id": format!("query-{event_type}-{payload_field}"),
18291                    "workflow_type": "missing",
18292                    "query_name": "missing",
18293                    "payload_codec": DEFAULT_CODEC,
18294                    "workflow_arguments": null,
18295                    "query_arguments": null,
18296                    "history_events": [{
18297                        "event_type": event_type,
18298                        "payload": payload
18299                    }]
18300                }))
18301                .expect("query task");
18302                let failure = worker
18303                    .execute_query_task(query)
18304                    .await
18305                    .expect_err("signal payload codec must precede unknown query outcome");
18306                assert_eq!(
18307                    failure.reason, "query_payload_decode_failed",
18308                    "{event_type}.{payload_field} returned an unrelated query outcome"
18309                );
18310                assert!(
18311                    failure.message.contains("unsupported_payload_codec"),
18312                    "{event_type}.{payload_field} returned an unrelated query error: {}",
18313                    failure.message
18314                );
18315            }
18316        }
18317    }
18318
18319    #[test]
18320    fn workflow_context_schedules_activity_until_completion_is_in_history() {
18321        let ctx = WorkflowContext {
18322            state: Arc::new(Mutex::new(
18323                WorkflowState::new_with_identity(
18324                    Vec::new(),
18325                    Some("wf-parent".to_string()),
18326                    Some("run-parent".to_string()),
18327                    "rust-workers".to_string(),
18328                    DEFAULT_CODEC.to_string(),
18329                    None,
18330                )
18331                .expect("workflow state"),
18332            )),
18333        };
18334
18335        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18336        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18337        assert!(matches!(
18338            call.as_mut().poll(&mut task_context),
18339            Poll::Pending
18340        ));
18341
18342        let commands = ctx.take_commands().expect("commands");
18343        assert_eq!(commands[0]["type"], "schedule_activity");
18344        assert_eq!(commands[0]["activity_type"], "hello.activity");
18345    }
18346
18347    #[test]
18348    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18349        let ctx = workflow_context(Vec::new());
18350        let options = ActivityOptions::new()
18351            .task_queue("payments")
18352            .retry_policy(
18353                ActivityRetryPolicy::new(4)
18354                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18355                    .non_retryable_error_type("ValidationError"),
18356            )
18357            .start_to_close_timeout(Duration::from_secs(120))
18358            .schedule_to_start_timeout(Duration::from_secs(10))
18359            .schedule_to_close_timeout(Duration::from_secs(300))
18360            .heartbeat_timeout(Duration::from_secs(15));
18361        let mut call = Box::pin(ctx.activity_with_options(
18362            "charge-card",
18363            options,
18364            json!([{"order_id": "o-1"}]),
18365        ));
18366        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18367
18368        assert!(matches!(
18369            call.as_mut().poll(&mut task_context),
18370            Poll::Pending
18371        ));
18372        assert!(matches!(
18373            call.as_mut().poll(&mut task_context),
18374            Poll::Pending
18375        ));
18376
18377        let commands = ctx.take_commands().expect("activity command");
18378        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18379        assert_eq!(commands[0]["queue"], "payments");
18380        assert_eq!(
18381            commands[0]["retry_policy"],
18382            json!({
18383                "max_attempts": 4,
18384                "backoff_seconds": [1, 3, 9],
18385                "non_retryable_error_types": ["ValidationError"],
18386            })
18387        );
18388        assert_eq!(commands[0]["start_to_close_timeout"], 120);
18389        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18390        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18391        assert_eq!(commands[0]["heartbeat_timeout"], 15);
18392    }
18393
18394    #[test]
18395    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18396        let ctx = workflow_context(Vec::new());
18397        let options = ActivityOptions::new().retry_policy(
18398            ActivityRetryPolicy::new(3)
18399                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18400        );
18401        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18402        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18403
18404        assert!(matches!(
18405            call.as_mut().poll(&mut task_context),
18406            Poll::Pending
18407        ));
18408        assert_eq!(
18409            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18410            json!([1, 2])
18411        );
18412    }
18413
18414    #[test]
18415    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18416        let cases = [
18417            (
18418                ActivityOptions::new().task_queue("  "),
18419                ActivityOptionsErrorKind::EmptyTaskQueue,
18420            ),
18421            (
18422                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18423                ActivityOptionsErrorKind::EmptyRetryPolicy,
18424            ),
18425            (
18426                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18427                ActivityOptionsErrorKind::InvalidMaxAttempts,
18428            ),
18429            (
18430                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18431                    max_attempts: None,
18432                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18433                    non_retryable_error_types: Vec::new(),
18434                }),
18435                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18436            ),
18437            (
18438                ActivityOptions::new().retry_policy(
18439                    ActivityRetryPolicy::new(2)
18440                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18441                ),
18442                ActivityOptionsErrorKind::TooManyBackoffIntervals,
18443            ),
18444            (
18445                ActivityOptions::new().retry_policy(
18446                    ActivityRetryPolicy::new(2).exponential_backoff(
18447                        Duration::from_secs(1),
18448                        0,
18449                        None,
18450                    ),
18451                ),
18452                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18453            ),
18454            (
18455                ActivityOptions::new()
18456                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
18457                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18458            ),
18459            (
18460                ActivityOptions::new().retry_policy(
18461                    ActivityRetryPolicy::new(10_002).exponential_backoff(
18462                        Duration::from_secs(1),
18463                        1,
18464                        None,
18465                    ),
18466                ),
18467                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18468            ),
18469            (
18470                ActivityOptions::new().retry_policy(
18471                    ActivityRetryPolicy::new(2)
18472                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18473                ),
18474                ActivityOptionsErrorKind::BackoffOverflow,
18475            ),
18476        ];
18477
18478        for (options, expected_kind) in cases {
18479            let ctx = workflow_context(Vec::new());
18480            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18481            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18482            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18483                call.as_mut().poll(&mut task_context)
18484            else {
18485                panic!("expected typed activity validation error");
18486            };
18487            assert_eq!(error.kind, expected_kind);
18488            assert!(ctx.take_commands().expect("commands").is_empty());
18489        }
18490    }
18491
18492    #[test]
18493    fn activity_options_validate_positive_and_ordered_timeouts() {
18494        let zero_timeout_cases = [
18495            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18496            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18497            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18498            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18499        ];
18500        for options in zero_timeout_cases {
18501            assert_eq!(
18502                options.validate().expect_err("zero timeout").kind,
18503                ActivityOptionsErrorKind::TimeoutNotPositive
18504            );
18505        }
18506
18507        let ordering_cases = [
18508            ActivityOptions::new()
18509                .heartbeat_timeout(Duration::from_secs(11))
18510                .start_to_close_timeout(Duration::from_secs(10)),
18511            ActivityOptions::new()
18512                .start_to_close_timeout(Duration::from_secs(31))
18513                .schedule_to_close_timeout(Duration::from_secs(30)),
18514            ActivityOptions::new()
18515                .schedule_to_start_timeout(Duration::from_secs(31))
18516                .schedule_to_close_timeout(Duration::from_secs(30)),
18517        ];
18518        for options in ordering_cases {
18519            assert_eq!(
18520                options.validate().expect_err("timeout order").kind,
18521                ActivityOptionsErrorKind::TimeoutOrder
18522            );
18523        }
18524
18525        assert_eq!(
18526            ActivityOptions::new()
18527                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18528                .validate()
18529                .expect_err("protocol integer overflow")
18530                .kind,
18531            ActivityOptionsErrorKind::TimeoutOverflow
18532        );
18533    }
18534
18535    #[test]
18536    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18537        let ctx = workflow_context(completed_retry_activity_history());
18538        let mut call =
18539            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18540        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18541
18542        assert!(matches!(
18543            call.as_mut().poll(&mut task_context),
18544            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18545        ));
18546        assert!(ctx.take_commands().expect("commands").is_empty());
18547        ctx.ensure_history_consumed().expect("history consumed");
18548    }
18549
18550    #[test]
18551    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18552        let mut options = retry_activity_options();
18553        options
18554            .retry_policy
18555            .as_mut()
18556            .expect("retry policy")
18557            .non_retryable_error_types
18558            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18559
18560        let new_ctx = workflow_context(Vec::new());
18561        let mut new_call =
18562            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18563        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18564        assert!(matches!(
18565            new_call.as_mut().poll(&mut task_context),
18566            Poll::Pending
18567        ));
18568        let commands = new_ctx.take_commands().expect("commands");
18569        assert_eq!(commands.len(), 1);
18570        assert_eq!(
18571            commands[0]["retry_policy"]["non_retryable_error_types"],
18572            json!(["PermanentError"])
18573        );
18574
18575        let replay_ctx = workflow_context(completed_retry_activity_history());
18576        let mut replay_call =
18577            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18578        assert!(matches!(
18579            replay_call.as_mut().poll(&mut task_context),
18580            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18581        ));
18582        assert!(replay_ctx.take_commands().expect("commands").is_empty());
18583        replay_ctx
18584            .ensure_history_consumed()
18585            .expect("history consumed");
18586    }
18587
18588    #[test]
18589    fn replayed_intermediate_retry_remains_pending_across_restarts() {
18590        let history = completed_retry_activity_history()
18591            .into_iter()
18592            .take(3)
18593            .collect::<Vec<_>>();
18594
18595        for _restart in 0..2 {
18596            let ctx = workflow_context(history.clone());
18597            let mut call =
18598                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18599            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18600            assert!(matches!(
18601                call.as_mut().poll(&mut task_context),
18602                Poll::Pending
18603            ));
18604            assert!(ctx.take_commands().expect("commands").is_empty());
18605        }
18606    }
18607
18608    #[test]
18609    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18610        let mut changed_queue = retry_activity_options();
18611        changed_queue.task_queue = Some("different-queue".to_string());
18612
18613        let mut changed_max_attempts = retry_activity_options();
18614        let retry_policy = changed_max_attempts
18615            .retry_policy
18616            .as_mut()
18617            .expect("retry policy");
18618        retry_policy.max_attempts = Some(4);
18619
18620        let mut changed_backoff = retry_activity_options();
18621        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18622        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18623            Duration::from_secs(3),
18624            Duration::from_secs(4),
18625        ]));
18626
18627        let mut changed_non_retryable_types = retry_activity_options();
18628        let retry_policy = changed_non_retryable_types
18629            .retry_policy
18630            .as_mut()
18631            .expect("retry policy");
18632        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18633
18634        let mut changed_start_to_close = retry_activity_options();
18635        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18636        let mut changed_schedule_to_start = retry_activity_options();
18637        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18638        let mut changed_schedule_to_close = retry_activity_options();
18639        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18640        let mut changed_heartbeat = retry_activity_options();
18641        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18642
18643        let cases = [
18644            (changed_queue, "activity_task_queue_mismatch"),
18645            (changed_max_attempts, "activity_retry_policy_mismatch"),
18646            (changed_backoff, "activity_retry_policy_mismatch"),
18647            (
18648                changed_non_retryable_types,
18649                "activity_retry_policy_mismatch",
18650            ),
18651            (changed_start_to_close, "activity_retry_policy_mismatch"),
18652            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18653            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18654            (changed_heartbeat, "activity_retry_policy_mismatch"),
18655        ];
18656
18657        for (options, expected_reason) in cases {
18658            let ctx = workflow_context(completed_retry_activity_history());
18659            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18660            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18661            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18662                call.as_mut().poll(&mut task_context)
18663            else {
18664                panic!("changed activity options must fail replay");
18665            };
18666            assert_eq!(failure.reason, expected_reason);
18667            assert_eq!(failure.sequence, Some(1));
18668            assert!(ctx.take_commands().expect("commands").is_empty());
18669        }
18670    }
18671
18672    #[test]
18673    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18674        let cases = [
18675            (
18676                "execution_mode",
18677                json!("local"),
18678                "activity_execution_mode_mismatch",
18679            ),
18680            (
18681                "snapshot_version",
18682                json!(2),
18683                "activity_retry_policy_mismatch",
18684            ),
18685        ];
18686
18687        for (field, value, expected_reason) in cases {
18688            let mut history = completed_retry_activity_history();
18689            let activity = history[0].payload["activity"]
18690                .as_object_mut()
18691                .expect("activity snapshot");
18692            if field == "execution_mode" {
18693                activity.insert(field.to_string(), value);
18694            } else {
18695                activity["retry_policy"]
18696                    .as_object_mut()
18697                    .expect("retry snapshot")
18698                    .insert(field.to_string(), value);
18699            }
18700
18701            let ctx = workflow_context(history);
18702            let mut call =
18703                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18704            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18705            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18706                call.as_mut().poll(&mut task_context)
18707            else {
18708                panic!("changed {field} must fail replay");
18709            };
18710            assert_eq!(failure.reason, expected_reason);
18711            assert_eq!(failure.sequence, Some(1));
18712            assert!(ctx.take_commands().expect("commands").is_empty());
18713        }
18714    }
18715
18716    #[test]
18717    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18718        let mut history = completed_retry_activity_history();
18719        let activity = history[0].payload["activity"]
18720            .as_object_mut()
18721            .expect("activity snapshot");
18722        activity.remove("execution_mode");
18723        activity.remove("retry_policy");
18724
18725        let mut current = retry_activity_options();
18726        current.start_to_close_timeout = Some(Duration::from_secs(45));
18727        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18728        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18729        current.heartbeat_timeout = Some(Duration::from_secs(12));
18730
18731        let ctx = workflow_context(history);
18732        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18733        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18734        assert!(matches!(
18735            call.as_mut().poll(&mut task_context),
18736            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18737        ));
18738        assert!(ctx.take_commands().expect("commands").is_empty());
18739        ctx.ensure_history_consumed().expect("history consumed");
18740    }
18741
18742    #[test]
18743    fn terminal_activity_failed_after_start_returns_typed_failure() {
18744        let history = vec![
18745            history_event(
18746                "ActivityScheduled",
18747                json!({
18748                    "sequence": 1,
18749                    "activity_type": "flaky",
18750                    "activity_execution_id": "act-terminal",
18751                    "activity": {
18752                        "id": "act-terminal",
18753                        "sequence": 1,
18754                        "type": "flaky",
18755                        "queue": "critical-activities",
18756                        "retry_policy": {
18757                            "snapshot_version": 1,
18758                            "max_attempts": 3,
18759                            "backoff_seconds": [2, 4],
18760                            "non_retryable_error_types": ["PermanentError"]
18761                        }
18762                    }
18763                }),
18764            ),
18765            history_event(
18766                "ActivityStarted",
18767                json!({
18768                    "sequence": 1,
18769                    "activity_type": "flaky",
18770                    "activity_execution_id": "act-terminal",
18771                    "activity_attempt_id": "attempt-1",
18772                    "attempt_number": 1
18773                }),
18774            ),
18775            history_event(
18776                "ActivityFailed",
18777                json!({
18778                    "sequence": 1,
18779                    "activity_type": "flaky",
18780                    "activity_execution_id": "act-terminal",
18781                    "activity_attempt_id": "attempt-1",
18782                    "attempt_number": 1,
18783                    "failure_id": "failure-terminal",
18784                    "failure_category": "activity",
18785                    "exception_type": "PermanentError",
18786                    "message": "cannot retry",
18787                    "non_retryable": true
18788                }),
18789            ),
18790        ];
18791        let ctx = workflow_context(history);
18792        let mut call =
18793            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18794        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18795
18796        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18797            call.as_mut().poll(&mut task_context)
18798        else {
18799            panic!("terminal ActivityFailed must settle the activity future");
18800        };
18801        assert_eq!(failure.kind, ActivityFailureKind::Failed);
18802        assert_eq!(
18803            failure.activity_execution_id.as_deref(),
18804            Some("act-terminal")
18805        );
18806        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18807        assert!(failure.non_retryable);
18808        assert!(ctx.take_commands().expect("commands").is_empty());
18809        ctx.ensure_history_consumed().expect("history consumed");
18810    }
18811
18812    #[test]
18813    fn activity_terminal_events_return_machine_readable_failures() {
18814        let cases = [
18815            (
18816                "ActivityFailed",
18817                json!({
18818                    "sequence": 1,
18819                    "activity_type": "charge-card",
18820                    "activity_execution_id": "act-1",
18821                    "activity_attempt_id": "attempt-2",
18822                    "attempt_number": 2,
18823                    "failure_id": "failure-1",
18824                    "failure_category": "activity",
18825                    "exception_type": "PaymentDeclined",
18826                    "exception_class": "payments.PaymentDeclined",
18827                    "message": "card declined",
18828                    "non_retryable": true
18829                }),
18830                ActivityFailureKind::Failed,
18831                "activity",
18832            ),
18833            (
18834                "ActivityCancelled",
18835                json!({
18836                    "sequence": 1,
18837                    "activity_type": "charge-card",
18838                    "activity_execution_id": "act-1",
18839                    "activity_attempt_id": "attempt-1"
18840                }),
18841                ActivityFailureKind::Cancelled,
18842                "cancelled",
18843            ),
18844        ];
18845
18846        for (event_type, payload, expected_kind, expected_reason) in cases {
18847            let ctx = workflow_context(vec![history_event(event_type, payload)]);
18848            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18849            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18850            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18851                call.as_mut().poll(&mut task_context)
18852            else {
18853                panic!("expected terminal activity failure");
18854            };
18855            assert_eq!(failure.kind, expected_kind);
18856            assert_eq!(failure.reason, expected_reason);
18857            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18858            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18859        }
18860    }
18861
18862    #[test]
18863    fn every_activity_timeout_class_is_typed() {
18864        for timeout_kind in [
18865            "start_to_close",
18866            "schedule_to_start",
18867            "schedule_to_close",
18868            "heartbeat",
18869        ] {
18870            let ctx = workflow_context(vec![history_event(
18871                "ActivityTimedOut",
18872                json!({
18873                    "sequence": 1,
18874                    "activity_type": "slow",
18875                    "activity_execution_id": "act-timeout",
18876                    "activity_attempt_id": "attempt-timeout",
18877                    "failure_category": "timeout",
18878                    "timeout_kind": timeout_kind,
18879                    "message": "deadline expired"
18880                }),
18881            )]);
18882            let mut call = Box::pin(ctx.activity("slow", json!([])));
18883            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18884            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18885                call.as_mut().poll(&mut task_context)
18886            else {
18887                panic!("expected timeout failure");
18888            };
18889            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18890            assert_eq!(failure.reason, timeout_kind);
18891            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18892            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18893        }
18894    }
18895
18896    #[test]
18897    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18898        let ctx = workflow_context(Vec::new());
18899        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18900        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18901
18902        assert!(matches!(
18903            sleep.as_mut().poll(&mut task_context),
18904            Poll::Pending
18905        ));
18906        assert!(matches!(
18907            sleep.as_mut().poll(&mut task_context),
18908            Poll::Pending
18909        ));
18910
18911        let commands = ctx.take_commands().expect("timer command");
18912        assert_eq!(
18913            commands,
18914            vec![json!({
18915                "type": "start_timer",
18916                "delay_seconds": 2,
18917            })]
18918        );
18919    }
18920
18921    #[test]
18922    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18923        let history = vec![
18924            history_event(
18925                "TimerScheduled",
18926                json!({
18927                    "sequence": 1,
18928                    "timer_id": "timer-1",
18929                    "delay_seconds": 5,
18930                    "fire_at": "2026-07-11T12:00:05Z",
18931                }),
18932            ),
18933            history_event(
18934                "TimerFired",
18935                json!({
18936                    "sequence": 1,
18937                    "timer_id": "timer-1",
18938                    "delay_seconds": 5,
18939                    "fire_at": "2026-07-11T12:00:05Z",
18940                    "fired_at": "2026-07-11T12:00:05Z",
18941                }),
18942            ),
18943        ];
18944
18945        for _restart in 0..2 {
18946            let ctx = workflow_context(history.clone());
18947            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18948            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18949            assert!(matches!(
18950                sleep.as_mut().poll(&mut task_context),
18951                Poll::Ready(Ok(()))
18952            ));
18953            assert!(ctx.take_commands().expect("commands").is_empty());
18954            ctx.ensure_history_consumed().expect("history consumed");
18955        }
18956    }
18957
18958    #[test]
18959    fn workflow_sleep_rejects_changed_delay_during_replay() {
18960        let ctx = workflow_context(vec![
18961            history_event(
18962                "TimerScheduled",
18963                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18964            ),
18965            history_event(
18966                "TimerFired",
18967                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18968            ),
18969        ]);
18970        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18971        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18972
18973        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18974            sleep.as_mut().poll(&mut task_context)
18975        else {
18976            panic!("changed timer delay must be rejected");
18977        };
18978        assert_eq!(failure.reason, "timer_delay_mismatch");
18979        assert_eq!(failure.sequence, Some(1));
18980    }
18981
18982    #[test]
18983    fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18984        let ctx = workflow_context(Vec::new());
18985        let mut wait = Box::pin(
18986            ctx.wait_condition(
18987                ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18988                    .timeout(Duration::from_millis(60_001)),
18989                || Ok(false),
18990            ),
18991        );
18992        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18993
18994        assert!(matches!(
18995            wait.as_mut().poll(&mut task_context),
18996            Poll::Pending
18997        ));
18998        assert!(matches!(
18999            wait.as_mut().poll(&mut task_context),
19000            Poll::Pending
19001        ));
19002        assert_eq!(
19003            ctx.take_commands().expect("condition command"),
19004            vec![json!({
19005                "type": "open_condition_wait",
19006                "condition_wait_occurrence_id": "rust:condition-wait:0",
19007                "condition_key": "approval.ready",
19008                "condition_definition_fingerprint": "sha256:approval-v1",
19009                "timeout_seconds": 61,
19010            })]
19011        );
19012    }
19013
19014    #[test]
19015    fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
19016        let ctx = workflow_context(Vec::new());
19017        let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
19018        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19019        assert!(matches!(
19020            satisfied.as_mut().poll(&mut task_context),
19021            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19022        ));
19023
19024        let mut timed_out = Box::pin(wait_condition!(
19025            ctx,
19026            "no-wait",
19027            timeout: Duration::ZERO,
19028            || Ok(false),
19029        ));
19030        assert!(matches!(
19031            timed_out.as_mut().poll(&mut task_context),
19032            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
19033        ));
19034        assert!(ctx.take_commands().expect("commands").is_empty());
19035    }
19036
19037    #[test]
19038    fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
19039        let signal_history = vec![
19040            history_event(
19041                "ConditionWaitOpened",
19042                json!({
19043                    "sequence": 4,
19044                    "condition_wait_id": "condition:4",
19045                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19046                    "condition_key": "approval",
19047                    "condition_definition_fingerprint": "sha256:approval-v1",
19048                    "timeout_seconds": 30,
19049                }),
19050            ),
19051            history_event(
19052                "SignalReceived",
19053                json!({
19054                    "workflow_sequence": 4,
19055                    "signal_name": "approve",
19056                    "arguments": fixture_envelope(json!(["Ada"])),
19057                }),
19058            ),
19059        ];
19060        for _worker_before_or_after_restart in 0..2 {
19061            let ctx = workflow_context(signal_history.clone());
19062            let predicate_ctx = ctx.clone();
19063            let mut wait = Box::pin(
19064                ctx.wait_condition(
19065                    ConditionWaitOptions::new("approval", "sha256:approval-v1")
19066                        .timeout(Duration::from_secs(30)),
19067                    move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
19068                ),
19069            );
19070            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19071            assert!(matches!(
19072                wait.as_mut().poll(&mut task_context),
19073                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19074            ));
19075            assert!(ctx.take_commands().expect("commands").is_empty());
19076            ctx.ensure_history_consumed().expect("condition consumed");
19077        }
19078
19079        let update_history = vec![
19080            history_event(
19081                "ConditionWaitOpened",
19082                json!({
19083                    "sequence": 7,
19084                    "condition_wait_id": "condition:7",
19085                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19086                    "condition_key": "update-approval",
19087                    "condition_definition_fingerprint": "sha256:update-approval-v1",
19088                }),
19089            ),
19090            history_event(
19091                "UpdateApplied",
19092                json!({
19093                    "sequence": 7,
19094                    "update_id": "update-1",
19095                    "update_name": "approve",
19096                    "arguments": fixture_envelope(json!([true])),
19097                }),
19098            ),
19099        ];
19100        let ctx = workflow_context(update_history);
19101        let predicate_ctx = ctx.clone();
19102        let mut wait = Box::pin(ctx.wait_condition(
19103            ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
19104            move || {
19105                Ok(predicate_ctx
19106                    .updates("approve")?
19107                    .first()
19108                    .and_then(|arguments| arguments.first())
19109                    .and_then(Value::as_bool)
19110                    == Some(true))
19111            },
19112        ));
19113        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19114        assert!(matches!(
19115            wait.as_mut().poll(&mut task_context),
19116            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19117        ));
19118        assert!(ctx.take_commands().expect("commands").is_empty());
19119        ctx.ensure_history_consumed().expect("condition consumed");
19120    }
19121
19122    #[test]
19123    fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
19124        let open_history = vec![
19125            history_event(
19126                "ConditionWaitOpened",
19127                json!({
19128                    "sequence": 3,
19129                    "condition_wait_id": "condition:3",
19130                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19131                    "condition_key": "two-votes",
19132                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19133                    "timeout_seconds": 120,
19134                }),
19135            ),
19136            history_event(
19137                "SignalReceived",
19138                json!({
19139                    "workflow_sequence": 3,
19140                    "signal_name": "vote",
19141                    "arguments": fixture_envelope(json!(["first"])),
19142                }),
19143            ),
19144        ];
19145        for _worker_before_or_after_restart in 0..2 {
19146            let ctx = workflow_context(open_history.clone());
19147            let predicate_ctx = ctx.clone();
19148            let mut wait = Box::pin(
19149                ctx.wait_condition(
19150                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
19151                        .timeout(Duration::from_secs(120)),
19152                    move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19153                ),
19154            );
19155            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19156            assert!(matches!(
19157                wait.as_mut().poll(&mut task_context),
19158                Poll::Pending
19159            ));
19160            assert_eq!(
19161                ctx.take_commands().expect("reopened condition"),
19162                vec![json!({
19163                    "type": "open_condition_wait",
19164                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19165                    "condition_key": "two-votes",
19166                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19167                    "timeout_seconds": 120,
19168                })]
19169            );
19170        }
19171
19172        let satisfied_ctx = workflow_context(vec![
19173            history_event(
19174                "ConditionWaitOpened",
19175                json!({
19176                    "sequence": 5,
19177                    "condition_wait_id": "condition:5",
19178                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19179                    "condition_key": "approval",
19180                    "condition_definition_fingerprint": "sha256:approval-v1",
19181                }),
19182            ),
19183            history_event(
19184                "ConditionWaitSatisfied",
19185                json!({
19186                    "sequence": 5,
19187                    "condition_wait_id": "condition:5",
19188                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19189                    "condition_key": "approval",
19190                    "condition_definition_fingerprint": "sha256:approval-v1",
19191                }),
19192            ),
19193        ]);
19194        let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
19195            ConditionWaitOptions::new("approval", "sha256:approval-v1"),
19196            || Ok(false),
19197        ));
19198        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19199        assert!(matches!(
19200            satisfied.as_mut().poll(&mut task_context),
19201            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19202        ));
19203
19204        let timed_out_ctx = workflow_context(vec![
19205            history_event(
19206                "ConditionWaitOpened",
19207                json!({
19208                    "sequence": 8,
19209                    "condition_wait_id": "condition:8",
19210                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19211                    "condition_key": "approval-timeout",
19212                    "condition_definition_fingerprint": "sha256:approval-timeout-v1",
19213                    "timeout_seconds": 5,
19214                }),
19215            ),
19216            history_event(
19217                "TimerScheduled",
19218                json!({
19219                    "sequence": 9,
19220                    "timer_id": "condition-timer:9",
19221                    "timer_kind": "condition_timeout",
19222                    "condition_wait_id": "condition:8",
19223                    "delay_seconds": 5,
19224                }),
19225            ),
19226            history_event(
19227                "TimerFired",
19228                json!({
19229                    "sequence": 9,
19230                    "timer_id": "condition-timer:9",
19231                    "timer_kind": "condition_timeout",
19232                    "condition_wait_id": "condition:8",
19233                    "delay_seconds": 5,
19234                }),
19235            ),
19236        ]);
19237        let mut timed_out = Box::pin(
19238            timed_out_ctx.wait_condition(
19239                ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
19240                    .timeout(Duration::from_secs(5)),
19241                || Ok(true),
19242            ),
19243        );
19244        assert!(matches!(
19245            timed_out.as_mut().poll(&mut task_context),
19246            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
19247        ));
19248    }
19249
19250    #[test]
19251    fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
19252        let history = vec![
19253            history_event(
19254                "ConditionWaitOpened",
19255                json!({
19256                    "sequence": 3,
19257                    "condition_wait_id": "condition:3",
19258                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19259                    "condition_key": "two-votes",
19260                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19261                }),
19262            ),
19263            history_event(
19264                "SignalReceived",
19265                json!({
19266                    "workflow_sequence": 3,
19267                    "signal_name": "vote",
19268                    "arguments": fixture_envelope(json!(["first"])),
19269                }),
19270            ),
19271            history_event(
19272                "ConditionWaitSatisfied",
19273                json!({
19274                    "sequence": 3,
19275                    "condition_wait_id": "condition:3",
19276                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19277                    "condition_key": "two-votes",
19278                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19279                }),
19280            ),
19281            history_event(
19282                "ConditionWaitOpened",
19283                json!({
19284                    "sequence": 5,
19285                    "condition_wait_id": "condition:5",
19286                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19287                    "condition_key": "two-votes",
19288                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19289                }),
19290            ),
19291            history_event(
19292                "SignalReceived",
19293                json!({
19294                    "workflow_sequence": 5,
19295                    "signal_name": "vote",
19296                    "arguments": fixture_envelope(json!(["second"])),
19297                }),
19298            ),
19299            history_event(
19300                "ConditionWaitSatisfied",
19301                json!({
19302                    "sequence": 5,
19303                    "condition_wait_id": "condition:5",
19304                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19305                    "condition_key": "two-votes",
19306                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19307                }),
19308            ),
19309        ];
19310        for _cold_worker_or_restart in 0..2 {
19311            let ctx = workflow_context(history.clone());
19312            let predicate_ctx = ctx.clone();
19313            let mut wait = Box::pin(ctx.wait_condition(
19314                ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19315                move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19316            ));
19317            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19318
19319            assert!(matches!(
19320                wait.as_mut().poll(&mut task_context),
19321                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19322            ));
19323            assert!(ctx.take_commands().expect("commands").is_empty());
19324            ctx.ensure_history_consumed()
19325                .expect("every physical wait-open is consumed");
19326        }
19327    }
19328
19329    #[test]
19330    fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19331        let history = vec![
19332            history_event(
19333                "ConditionWaitOpened",
19334                json!({
19335                    "sequence": 3,
19336                    "condition_wait_id": "condition:3",
19337                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19338                    "condition_key": "approved",
19339                    "condition_definition_fingerprint": "sha256:approved-v1",
19340                }),
19341            ),
19342            history_event(
19343                "UpdateApplied",
19344                json!({
19345                    "sequence": 3,
19346                    "update_id": "update-1",
19347                    "update_name": "approve",
19348                    "arguments": fixture_envelope(json!([false])),
19349                }),
19350            ),
19351            history_event(
19352                "ConditionWaitOpened",
19353                json!({
19354                    "sequence": 5,
19355                    "condition_wait_id": "condition:5",
19356                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19357                    "condition_key": "approved",
19358                    "condition_definition_fingerprint": "sha256:approved-v1",
19359                }),
19360            ),
19361            history_event(
19362                "UpdateApplied",
19363                json!({
19364                    "sequence": 5,
19365                    "update_id": "update-2",
19366                    "update_name": "approve",
19367                    "arguments": fixture_envelope(json!([true])),
19368                }),
19369            ),
19370        ];
19371
19372        for _cold_worker_or_restart in 0..2 {
19373            let ctx = workflow_context(history.clone());
19374            let predicate_ctx = ctx.clone();
19375            let mut wait = Box::pin(ctx.wait_condition(
19376                ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19377                move || {
19378                    Ok(predicate_ctx
19379                        .updates("approve")?
19380                        .last()
19381                        .and_then(|arguments| arguments.first())
19382                        .and_then(Value::as_bool)
19383                        == Some(true))
19384                },
19385            ));
19386            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19387
19388            assert!(matches!(
19389                wait.as_mut().poll(&mut task_context),
19390                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19391            ));
19392            assert!(ctx.take_commands().expect("commands").is_empty());
19393            ctx.ensure_history_consumed()
19394                .expect("every update-driven reopen is consumed");
19395        }
19396    }
19397
19398    #[test]
19399    fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19400        for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19401            ("shared", "sha256:first", "shared", "sha256:second"),
19402            ("first", "sha256:shared", "second", "sha256:shared"),
19403            ("shared", "sha256:shared", "shared", "sha256:shared"),
19404            ("first", "sha256:first", "second", "sha256:second"),
19405        ] {
19406            let history = vec![
19407                history_event(
19408                    "ConditionWaitOpened",
19409                    json!({
19410                        "sequence": 3,
19411                        "condition_wait_id": "condition:3",
19412                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19413                        "condition_key": first_key,
19414                        "condition_definition_fingerprint": first_fingerprint,
19415                    }),
19416                ),
19417                history_event(
19418                    "ConditionWaitSatisfied",
19419                    json!({
19420                        "sequence": 3,
19421                        "condition_wait_id": "condition:3",
19422                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19423                        "condition_key": first_key,
19424                        "condition_definition_fingerprint": first_fingerprint,
19425                    }),
19426                ),
19427                history_event(
19428                    "ConditionWaitOpened",
19429                    json!({
19430                        "sequence": 4,
19431                        "condition_wait_id": "condition:4",
19432                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19433                        "condition_key": second_key,
19434                        "condition_definition_fingerprint": second_fingerprint,
19435                    }),
19436                ),
19437                history_event(
19438                    "ConditionWaitSatisfied",
19439                    json!({
19440                        "sequence": 4,
19441                        "condition_wait_id": "condition:4",
19442                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19443                        "condition_key": second_key,
19444                        "condition_definition_fingerprint": second_fingerprint,
19445                    }),
19446                ),
19447            ];
19448            for _cold_worker_or_restart in 0..2 {
19449                let ctx = workflow_context(history.clone());
19450                let mut task_context = TaskContext::from_waker(noop_waker_ref());
19451                let mut first = Box::pin(ctx.wait_condition(
19452                    ConditionWaitOptions::new(first_key, first_fingerprint),
19453                    || Ok(false),
19454                ));
19455                assert!(matches!(
19456                    first.as_mut().poll(&mut task_context),
19457                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19458                ));
19459
19460                let mut second = Box::pin(ctx.wait_condition(
19461                    ConditionWaitOptions::new(second_key, second_fingerprint),
19462                    || Ok(false),
19463                ));
19464                assert!(matches!(
19465                    second.as_mut().poll(&mut task_context),
19466                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19467                ));
19468                assert!(ctx.take_commands().expect("commands").is_empty());
19469                ctx.ensure_history_consumed()
19470                    .expect("each authored wait consumes one occurrence");
19471            }
19472        }
19473    }
19474
19475    #[test]
19476    fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19477        fn worker() -> Worker {
19478            let client = Client::new("http://127.0.0.1:8080").expect("client");
19479            let mut worker = Worker::new(client, "rust-workers");
19480            worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19481                let mut outcomes = Vec::new();
19482                for _ in 0..2 {
19483                    outcomes.push(
19484                        ctx.wait_condition(
19485                            ConditionWaitOptions::new("shared", "sha256:shared"),
19486                            || Ok(false),
19487                        )
19488                        .await?,
19489                    );
19490                }
19491                Ok(json!(outcomes))
19492            });
19493            worker
19494        }
19495
19496        let task = workflow_task(
19497            "rust.condition-loop",
19498            vec![
19499                history_event(
19500                    "ConditionWaitOpened",
19501                    json!({
19502                        "sequence": 1,
19503                        "condition_wait_id": "condition:1",
19504                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19505                        "condition_key": "shared",
19506                        "condition_definition_fingerprint": "sha256:shared",
19507                    }),
19508                ),
19509                history_event(
19510                    "ConditionWaitSatisfied",
19511                    json!({
19512                        "sequence": 1,
19513                        "condition_wait_id": "condition:1",
19514                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19515                        "condition_key": "shared",
19516                        "condition_definition_fingerprint": "sha256:shared",
19517                    }),
19518                ),
19519                history_event(
19520                    "ConditionWaitOpened",
19521                    json!({
19522                        "sequence": 2,
19523                        "condition_wait_id": "condition:2",
19524                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19525                        "condition_key": "shared",
19526                        "condition_definition_fingerprint": "sha256:shared",
19527                    }),
19528                ),
19529                history_event(
19530                    "ConditionWaitSatisfied",
19531                    json!({
19532                        "sequence": 2,
19533                        "condition_wait_id": "condition:2",
19534                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19535                        "condition_key": "shared",
19536                        "condition_definition_fingerprint": "sha256:shared",
19537                    }),
19538                ),
19539            ],
19540            DEFAULT_CODEC,
19541        );
19542
19543        for _cold_worker_or_restart in 0..2 {
19544            let commands = worker()
19545                .execute_workflow_task(task.clone())
19546                .expect("adjacent loop waits replay deterministically");
19547            assert_eq!(commands.len(), 1);
19548            assert_eq!(commands[0]["type"], "complete_workflow");
19549            assert_eq!(
19550                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19551                json!(["satisfied", "satisfied"])
19552            );
19553        }
19554    }
19555
19556    #[test]
19557    fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19558        let history = vec![history_event(
19559            "ConditionWaitOpened",
19560            json!({
19561                "sequence": 12,
19562                "condition_wait_id": "condition:12",
19563                "condition_wait_occurrence_id": "rust:condition-wait:0",
19564                "condition_key": "approval",
19565                "condition_definition_fingerprint": "sha256:approval-v1",
19566                "timeout_seconds": 30,
19567            }),
19568        )];
19569        for (options, expected_reason) in [
19570            (
19571                ConditionWaitOptions::new("changed", "sha256:approval-v1")
19572                    .timeout(Duration::from_secs(30)),
19573                "condition_wait_key_mismatch",
19574            ),
19575            (
19576                ConditionWaitOptions::new("approval", "sha256:approval-v2")
19577                    .timeout(Duration::from_secs(30)),
19578                "condition_wait_predicate_mismatch",
19579            ),
19580            (
19581                ConditionWaitOptions::new("approval", "sha256:approval-v1")
19582                    .timeout(Duration::from_secs(29)),
19583                "condition_wait_timeout_mismatch",
19584            ),
19585        ] {
19586            let ctx = workflow_context(history.clone());
19587            let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19588            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19589            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19590                wait.as_mut().poll(&mut task_context)
19591            else {
19592                panic!("changed condition definition must fail replay");
19593            };
19594            assert_eq!(failure.reason, expected_reason);
19595            assert_eq!(failure.sequence, Some(12));
19596        }
19597    }
19598
19599    #[test]
19600    fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19601        let error = WorkflowState::new(
19602            vec![history_event(
19603                "ConditionWaitOpened",
19604                json!({
19605                    "sequence": 12,
19606                    "condition_wait_id": "condition:12",
19607                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19608                    "condition_key": "approval",
19609                }),
19610            )],
19611            "rust-workers".to_string(),
19612            DEFAULT_CODEC.to_string(),
19613            None,
19614        )
19615        .expect_err("condition history without a predicate fingerprint must fail");
19616
19617        assert!(matches!(
19618            error,
19619            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19620                if reason == "condition_wait_predicate_fingerprint_missing"
19621        ));
19622    }
19623
19624    #[test]
19625    fn condition_wait_history_requires_authored_occurrence_identity() {
19626        let error = WorkflowState::new(
19627            vec![history_event(
19628                "ConditionWaitOpened",
19629                json!({
19630                    "sequence": 12,
19631                    "condition_wait_id": "condition:12",
19632                    "condition_key": "approval",
19633                    "condition_definition_fingerprint": "sha256:approval-v1",
19634                }),
19635            )],
19636            "rust-workers".to_string(),
19637            DEFAULT_CODEC.to_string(),
19638            None,
19639        )
19640        .expect_err("condition history without occurrence identity must fail");
19641
19642        assert!(matches!(
19643            error,
19644            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19645                if reason == "condition_wait_occurrence_id_missing"
19646        ));
19647    }
19648
19649    #[test]
19650    fn typed_search_attribute_updates_validate_emit_and_replay() {
19651        let update = SearchAttributeUpdate::new()
19652            .keyword("OrderStatus", " waiting ")
19653            .expect("keyword")
19654            .int("Attempt", 3)
19655            .expect("int")
19656            .bool("Escalated", false)
19657            .expect("bool")
19658            .keyword_list("Regions", ["us-east", "eu-west"])
19659            .expect("list")
19660            .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19661            .expect("datetime")
19662            .delete("LegacyStatus")
19663            .expect("delete");
19664        let ctx = workflow_context(Vec::new());
19665        ctx.upsert_search_attributes(update.clone())
19666            .expect("typed update");
19667        assert_eq!(
19668            ctx.take_commands().expect("search-attribute command"),
19669            vec![json!({
19670                "type": "upsert_search_attributes",
19671                "attributes": {
19672                    "Attempt": 3,
19673                    "Escalated": false,
19674                    "LegacyStatus": null,
19675                    "OrderStatus": "waiting",
19676                    "Regions": ["us-east", "eu-west"],
19677                    "UpdatedAt": "2026-08-22T04:00:00Z",
19678                },
19679                "attribute_types": {
19680                    "Attempt": "int",
19681                    "Escalated": "bool",
19682                    "OrderStatus": "keyword",
19683                    "Regions": "keyword_list",
19684                    "UpdatedAt": "datetime",
19685                },
19686            })]
19687        );
19688
19689        let replay = workflow_context(vec![history_event(
19690            "SearchAttributesUpserted",
19691            json!({
19692                "sequence": 6,
19693                "attributes": {
19694                    "Attempt": 3,
19695                    "Escalated": false,
19696                    "LegacyStatus": null,
19697                    "OrderStatus": "waiting",
19698                    "Regions": ["us-east", "eu-west"],
19699                    "UpdatedAt": "2026-08-22T04:00:00Z",
19700                },
19701                "attribute_types": {
19702                    "Attempt": "int",
19703                    "Escalated": "bool",
19704                    "OrderStatus": "keyword",
19705                    "Regions": "keyword_list",
19706                    "UpdatedAt": "datetime",
19707                },
19708                "merged": {},
19709            }),
19710        )]);
19711        replay
19712            .upsert_search_attributes(update)
19713            .expect("matching update replays");
19714        assert!(replay.take_commands().expect("commands").is_empty());
19715        replay.ensure_history_consumed().expect("history consumed");
19716
19717        let type_drift = workflow_context(vec![history_event(
19718            "SearchAttributesUpserted",
19719            json!({
19720                "sequence": 7,
19721                "attributes": {"OrderStatus": "waiting"},
19722                "attribute_types": {"OrderStatus": "keyword"},
19723                "merged": {"OrderStatus": "waiting"},
19724            }),
19725        )]);
19726        let error = type_drift
19727            .upsert_search_attributes(
19728                SearchAttributeUpdate::new()
19729                    .string("OrderStatus", "waiting")
19730                    .expect("string update"),
19731            )
19732            .expect_err("same JSON value with a changed type must fail replay");
19733        assert!(matches!(
19734            error,
19735            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19736                if reason == "search_attribute_type_mismatch"
19737        ));
19738
19739        let malformed_types = WorkflowState::new(
19740            vec![history_event(
19741                "SearchAttributesUpserted",
19742                json!({
19743                    "sequence": 8,
19744                    "attributes": {"OrderStatus": "waiting"},
19745                    "attribute_types": {"OrderStatus": "unsupported"},
19746                    "merged": {"OrderStatus": "waiting"},
19747                }),
19748            )],
19749            "rust-workers".to_string(),
19750            DEFAULT_CODEC.to_string(),
19751            None,
19752        )
19753        .expect_err("unsupported search-attribute type metadata must fail");
19754        assert!(matches!(
19755            malformed_types,
19756            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19757                if reason == "search_attribute_types_malformed"
19758        ));
19759
19760        assert!(matches!(
19761            SearchAttributeUpdate::new().keyword("bad key", "value"),
19762            Err(SearchAttributeUpdateError::InvalidKey(_))
19763        ));
19764        assert!(matches!(
19765            SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19766            Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19767        ));
19768        assert!(matches!(
19769            SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19770            Err(SearchAttributeUpdateError::ValueTooLong { .. })
19771        ));
19772        assert!(matches!(
19773            SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19774            Err(SearchAttributeUpdateError::InvalidDateTime(_))
19775        ));
19776        assert!(matches!(
19777            workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19778            Err(Error::InvalidSearchAttributeUpdate(
19779                SearchAttributeUpdateError::Empty
19780            ))
19781        ));
19782    }
19783
19784    #[test]
19785    fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19786        let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19787        let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19788
19789        assert!(SearchAttributeUpdate::new()
19790            .string("AsciiDescription", ascii)
19791            .is_ok());
19792        assert!(SearchAttributeUpdate::new()
19793            .string("Utf8Description", utf8)
19794            .is_ok());
19795        assert!(matches!(
19796            SearchAttributeUpdate::new().string(
19797                "TooLongDescription",
19798                "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19799            ),
19800            Err(SearchAttributeUpdateError::ValueTooLong {
19801                kind: "string",
19802                limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19803                ..
19804            })
19805        ));
19806    }
19807
19808    #[test]
19809    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19810        let lone_fire = WorkflowState::new(
19811            vec![history_event(
19812                "TimerFired",
19813                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19814            )],
19815            "rust-workers".to_string(),
19816            DEFAULT_CODEC.to_string(),
19817            None,
19818        )
19819        .expect_err("TimerFired requires TimerScheduled");
19820        assert!(matches!(
19821            lone_fire,
19822            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19823                if reason == "timer_schedule_missing_or_duplicate"
19824        ));
19825
19826        let wrong_identity = WorkflowState::new(
19827            vec![
19828                history_event(
19829                    "TimerScheduled",
19830                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19831                ),
19832                history_event(
19833                    "TimerFired",
19834                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19835                ),
19836            ],
19837            "rust-workers".to_string(),
19838            DEFAULT_CODEC.to_string(),
19839            None,
19840        )
19841        .expect_err("fire must match scheduled timer identity");
19842        assert!(matches!(
19843            wrong_identity,
19844            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19845                if reason == "timer_identity_mismatch"
19846        ));
19847
19848        let duplicate_fire = WorkflowState::new(
19849            vec![
19850                history_event(
19851                    "TimerScheduled",
19852                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19853                ),
19854                history_event(
19855                    "TimerFired",
19856                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19857                ),
19858                history_event(
19859                    "TimerFired",
19860                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19861                ),
19862            ],
19863            "rust-workers".to_string(),
19864            DEFAULT_CODEC.to_string(),
19865            None,
19866        )
19867        .expect_err("a durable timer cannot fire twice");
19868        assert!(matches!(
19869            duplicate_fire,
19870            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19871                if reason == "duplicate_timer_fire"
19872        ));
19873
19874        let wrong_fired_delay = WorkflowState::new(
19875            vec![
19876                history_event(
19877                    "TimerScheduled",
19878                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19879                ),
19880                history_event(
19881                    "TimerFired",
19882                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19883                ),
19884            ],
19885            "rust-workers".to_string(),
19886            DEFAULT_CODEC.to_string(),
19887            None,
19888        )
19889        .expect_err("timer schedule and fire delays must agree");
19890        assert!(matches!(
19891            wrong_fired_delay,
19892            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19893                if reason == "timer_history_delay_mismatch"
19894        ));
19895    }
19896
19897    #[test]
19898    fn replay_rejects_activity_moved_before_recorded_timer() {
19899        let ctx = workflow_context(vec![
19900            history_event(
19901                "TimerScheduled",
19902                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19903            ),
19904            history_event(
19905                "TimerFired",
19906                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19907            ),
19908            history_event(
19909                "ActivityCompleted",
19910                json!({
19911                    "sequence": 2,
19912                    "activity_type": "after-timer",
19913                    "payload_codec": DEFAULT_CODEC,
19914                    "result": fixture_envelope(json!("done")),
19915                }),
19916            ),
19917        ]);
19918        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19919        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19920
19921        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19922            activity.as_mut().poll(&mut task_context)
19923        else {
19924            panic!("reordered durable command must be rejected");
19925        };
19926        assert_eq!(failure.reason, "recorded_command_mismatch");
19927        assert_eq!(failure.sequence, Some(1));
19928        assert_eq!(failure.expected.as_deref(), Some("timer"));
19929        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19930    }
19931
19932    #[test]
19933    fn workflow_context_emits_a_typed_named_signal_wait() {
19934        let ctx = workflow_context(Vec::new());
19935        let mut signal = Box::pin(ctx.wait_signal("finish"));
19936        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19937
19938        assert!(matches!(
19939            signal.as_mut().poll(&mut task_context),
19940            Poll::Pending
19941        ));
19942        assert_eq!(
19943            ctx.take_commands().expect("signal-wait command"),
19944            vec![json!({
19945                "type": "open_signal_wait",
19946                "signal_name": "finish",
19947            })]
19948        );
19949    }
19950
19951    #[test]
19952    fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19953        let ctx = workflow_context(Vec::new());
19954        let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19955        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19956
19957        let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19958        else {
19959            panic!("runtime-reserved signal should be rejected");
19960        };
19961        assert!(message.contains("reserved by the workflow runtime"));
19962        assert!(ctx.take_commands().expect("commands").is_empty());
19963    }
19964
19965    #[tokio::test]
19966    async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19967        let client = Client::builder("http://127.0.0.1:9")
19968            .build()
19969            .expect("client");
19970        let error = client
19971            .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19972            .await
19973            .expect_err("runtime-reserved signal should be rejected before transport");
19974
19975        assert!(
19976            matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19977        );
19978    }
19979
19980    #[test]
19981    fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19982        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19983            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19984                value.to_string(),
19985            )]))
19986            .expect("message payload");
19987            json!({
19988                "schema": MESSAGE_STREAM_SCHEMA,
19989                "stream_name": "orders",
19990                "message_id": message_id,
19991                "position": position,
19992                "payload_envelope": payload,
19993            })
19994        }
19995
19996        fn opened(sequence: u64) -> HistoryEvent {
19997            history_event(
19998                "SignalWaitOpened",
19999                json!({
20000                    "sequence": sequence,
20001                    "signal_name": MESSAGE_STREAM_SIGNAL,
20002                }),
20003            )
20004        }
20005
20006        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20007            history_event(
20008                "SignalApplied",
20009                json!({
20010                    "sequence": sequence,
20011                    "signal_name": MESSAGE_STREAM_SIGNAL,
20012                    "value": fixture_envelope(json!([delivery])),
20013                }),
20014            )
20015        }
20016
20017        fn received(delivery: Value) -> HistoryEvent {
20018            history_event(
20019                "SignalReceived",
20020                json!({
20021                    "signal_name": MESSAGE_STREAM_SIGNAL,
20022                    "arguments": fixture_envelope(json!([delivery])),
20023                    "payload_codec": DEFAULT_CODEC,
20024                }),
20025            )
20026        }
20027
20028        let client = Client::new("http://127.0.0.1:8080").expect("client");
20029        let mut worker = Worker::new(client, "rust-workers");
20030        worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
20031            let messages = ctx.message_stream("orders")?.receive(2).await?;
20032            Ok(json!(messages
20033                .into_iter()
20034                .map(|message| message.message_id)
20035                .collect::<Vec<_>>()))
20036        });
20037
20038        let first = delivery("message-1", 1, "one");
20039        let second = delivery("message-2", 2, "two");
20040        let batch = worker
20041            .execute_workflow_task_decision(workflow_task(
20042                "rust.message-stream-batch",
20043                vec![
20044                    opened(1),
20045                    received(first.clone()),
20046                    applied(1, first.clone()),
20047                    received(first.clone()),
20048                    received(second),
20049                ],
20050                DEFAULT_CODEC,
20051            ))
20052            .expect("worker task consumes the available batch");
20053
20054        assert_eq!(batch.commands.len(), 1);
20055        assert_eq!(batch.commands[0]["type"], "complete_workflow");
20056        assert_eq!(
20057            decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
20058                .expect("workflow result"),
20059            json!(["message-1", "message-2"])
20060        );
20061        assert_eq!(
20062            batch.message_stream_cursors,
20063            vec![json!({"stream_name": "orders", "through_position": 2})]
20064        );
20065        assert!(batch.message_stream_waits.is_empty());
20066
20067        let partial = worker
20068            .execute_workflow_task_decision(workflow_task(
20069                "rust.message-stream-batch",
20070                vec![opened(1), received(first.clone()), applied(1, first)],
20071                DEFAULT_CODEC,
20072            ))
20073            .expect("worker task returns without waiting for a missing second item");
20074        assert_eq!(partial.commands.len(), 1);
20075        assert_eq!(partial.commands[0]["type"], "complete_workflow");
20076        assert_eq!(
20077            decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
20078                .expect("workflow result"),
20079            json!(["message-1"])
20080        );
20081        assert_eq!(
20082            partial.message_stream_cursors,
20083            vec![json!({"stream_name": "orders", "through_position": 1})]
20084        );
20085        assert!(partial.message_stream_waits.is_empty());
20086    }
20087
20088    #[test]
20089    fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
20090        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20091            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20092                value.to_string(),
20093            )]))
20094            .expect("message payload");
20095            json!({
20096                "schema": MESSAGE_STREAM_SCHEMA,
20097                "stream_name": "orders",
20098                "message_id": message_id,
20099                "position": position,
20100                "payload_envelope": payload,
20101            })
20102        }
20103
20104        fn opened(sequence: u64) -> HistoryEvent {
20105            history_event(
20106                "SignalWaitOpened",
20107                json!({
20108                    "sequence": sequence,
20109                    "signal_name": MESSAGE_STREAM_SIGNAL,
20110                }),
20111            )
20112        }
20113
20114        fn received(delivery: Value) -> HistoryEvent {
20115            history_event(
20116                "SignalReceived",
20117                json!({
20118                    "signal_name": MESSAGE_STREAM_SIGNAL,
20119                    "arguments": fixture_envelope(json!([delivery])),
20120                    "payload_codec": DEFAULT_CODEC,
20121                }),
20122            )
20123        }
20124
20125        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20126            history_event(
20127                "SignalApplied",
20128                json!({
20129                    "sequence": sequence,
20130                    "signal_name": MESSAGE_STREAM_SIGNAL,
20131                    "value": fixture_envelope(json!([delivery])),
20132                }),
20133            )
20134        }
20135
20136        let client = Client::new("http://127.0.0.1:8080").expect("client");
20137        let mut worker = Worker::new(client, "rust-workers");
20138        worker.register_workflow(
20139            "rust.message-stream-partial-batches",
20140            |ctx, _input| async move {
20141                let stream = ctx.message_stream("orders")?;
20142                let first = stream.receive(10).await?;
20143                let second = stream.receive(10).await?;
20144                Ok(json!([
20145                    first
20146                        .into_iter()
20147                        .map(|message| message.message_id)
20148                        .collect::<Vec<_>>(),
20149                    second
20150                        .into_iter()
20151                        .map(|message| message.message_id)
20152                        .collect::<Vec<_>>(),
20153                ]))
20154            },
20155        );
20156
20157        let first = delivery("message-1", 1, "one");
20158        let second = delivery("message-2", 2, "two");
20159        let decision = worker
20160            .execute_workflow_task_decision(workflow_task(
20161                "rust.message-stream-partial-batches",
20162                vec![
20163                    opened(1),
20164                    received(first.clone()),
20165                    applied(1, first),
20166                    opened(2),
20167                    received(second.clone()),
20168                    applied(2, second),
20169                ],
20170                DEFAULT_CODEC,
20171            ))
20172            .expect("cold replay preserves both authored receive boundaries");
20173
20174        assert_eq!(decision.commands.len(), 1);
20175        assert_eq!(decision.commands[0]["type"], "complete_workflow");
20176        assert_eq!(
20177            decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
20178                .expect("workflow result"),
20179            json!([["message-1"], ["message-2"]])
20180        );
20181        assert_eq!(
20182            decision.message_stream_cursors,
20183            vec![json!({"stream_name": "orders", "through_position": 2})]
20184        );
20185        assert!(decision.message_stream_waits.is_empty());
20186    }
20187
20188    #[test]
20189    fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
20190        let ctx = workflow_context(Vec::new());
20191        let stream = ctx.message_stream("orders").expect("message stream");
20192        let mut receive = Box::pin(stream.receive(10));
20193        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20194
20195        assert!(matches!(
20196            receive.as_mut().poll(&mut task_context),
20197            Poll::Pending
20198        ));
20199        assert_eq!(
20200            ctx.take_commands().expect("message-stream wait command"),
20201            vec![json!({
20202                "type": "open_signal_wait",
20203                "signal_name": MESSAGE_STREAM_SIGNAL,
20204            })]
20205        );
20206        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20207        assert!(cursors.is_empty());
20208        assert_eq!(
20209            waits,
20210            vec![json!({"stream_name": "orders", "after_position": 0})]
20211        );
20212    }
20213
20214    #[test]
20215    fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
20216        let ctx = workflow_context(vec![history_event(
20217            "SignalReceived",
20218            json!({
20219                "signal_name": MESSAGE_STREAM_SIGNAL,
20220                "arguments": fixture_envelope(json!([{
20221                    "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
20222                    "stream_name": "orders",
20223                    "through_position": 2,
20224                }])),
20225                "payload_codec": DEFAULT_CODEC,
20226            }),
20227        )]);
20228        let stream = ctx.message_stream("orders").expect("message stream");
20229        let mut receive = Box::pin(stream.receive(10));
20230        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20231
20232        assert!(matches!(
20233            receive.as_mut().poll(&mut task_context),
20234            Poll::Pending
20235        ));
20236        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20237        assert_eq!(
20238            cursors,
20239            vec![json!({"stream_name": "orders", "through_position": 2})]
20240        );
20241        assert_eq!(
20242            waits,
20243            vec![json!({"stream_name": "orders", "after_position": 2})]
20244        );
20245    }
20246
20247    #[test]
20248    fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
20249        let mut empty_map = BTreeMap::new();
20250        let mut nested = BTreeMap::new();
20251        nested.insert(
20252            "value".to_string(),
20253            AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
20254        );
20255        let values = vec![
20256            AvroValue::Bytes(vec![0, 255]),
20257            AvroValue::Long(1),
20258            AvroValue::Double(1.0),
20259            AvroValue::Array(Vec::new()),
20260            AvroValue::Map(std::mem::take(&mut empty_map)),
20261            AvroValue::Map(nested),
20262        ];
20263        let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
20264        let transport = vec![json!({
20265            "schema": MESSAGE_STREAM_SCHEMA,
20266            "stream_name": "orders",
20267            "message_id": "message-1",
20268            "position": 1,
20269            "payload_envelope": payload,
20270        })];
20271
20272        for _ in 0..2 {
20273            let Some(MessageStreamDelivery::Message(message)) =
20274                decode_message_stream_delivery(transport.clone()).expect("delivery")
20275            else {
20276                panic!("message delivery expected");
20277            };
20278            assert_eq!(message.arguments, values);
20279            assert!(matches!(message.arguments[1], AvroValue::Long(1)));
20280            assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
20281        }
20282    }
20283
20284    #[test]
20285    fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
20286        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20287            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20288                value.to_string(),
20289            )]))
20290            .expect("message payload");
20291            json!({
20292                "schema": MESSAGE_STREAM_SCHEMA,
20293                "stream_name": "orders",
20294                "message_id": message_id,
20295                "position": position,
20296                "payload_envelope": payload,
20297            })
20298        }
20299
20300        fn opened(sequence: u64) -> HistoryEvent {
20301            history_event(
20302                "SignalWaitOpened",
20303                json!({
20304                    "sequence": sequence,
20305                    "signal_name": MESSAGE_STREAM_SIGNAL,
20306                }),
20307            )
20308        }
20309
20310        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20311            history_event(
20312                "SignalApplied",
20313                json!({
20314                    "sequence": sequence,
20315                    "signal_name": MESSAGE_STREAM_SIGNAL,
20316                    "value": fixture_envelope(json!([delivery])),
20317                }),
20318            )
20319        }
20320
20321        fn worker() -> Worker {
20322            let client = Client::new("http://127.0.0.1:8080").expect("client");
20323            let mut worker = Worker::new(client, "rust-workers");
20324            worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20325                let stream = ctx.message_stream("orders")?;
20326                let first = stream.receive_one().await?;
20327                let second = stream.receive_one().await?;
20328                Ok(json!([first.message_id, second.message_id]))
20329            });
20330            worker
20331        }
20332
20333        fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20334            let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20335            task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20336            task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20337            task
20338        }
20339
20340        let waiting = worker()
20341            .execute_workflow_task_decision(workflow_task(
20342                "rust.message-stream",
20343                Vec::new(),
20344                DEFAULT_CODEC,
20345            ))
20346            .expect("first worker opens the stream wait");
20347        assert_eq!(
20348            waiting.commands,
20349            vec![json!({
20350                "type": "open_signal_wait",
20351                "signal_name": MESSAGE_STREAM_SIGNAL,
20352            })]
20353        );
20354        assert!(waiting.message_stream_cursors.is_empty());
20355        assert_eq!(
20356            waiting.message_stream_waits,
20357            vec![json!({"stream_name": "orders", "after_position": 0})]
20358        );
20359
20360        let first_delivery = delivery("message-1", 1, "one");
20361        let first_arrival = worker()
20362            .execute_workflow_task_decision(task_with_resume(
20363                vec![opened(1)],
20364                first_delivery.clone(),
20365            ))
20366            .expect("replacement worker consumes the first arrival");
20367        assert_eq!(
20368            first_arrival.commands,
20369            vec![json!({
20370                "type": "open_signal_wait",
20371                "signal_name": MESSAGE_STREAM_SIGNAL,
20372            })]
20373        );
20374        assert_eq!(
20375            first_arrival.message_stream_cursors,
20376            vec![json!({"stream_name": "orders", "through_position": 1})]
20377        );
20378        assert_eq!(
20379            first_arrival.message_stream_waits,
20380            vec![json!({"stream_name": "orders", "after_position": 1})]
20381        );
20382
20383        let second_delivery = delivery("message-2", 2, "two");
20384        let first_applied = applied(1, first_delivery);
20385        let completed = worker()
20386            .execute_workflow_task_decision(task_with_resume(
20387                vec![opened(1), first_applied.clone(), opened(2)],
20388                second_delivery.clone(),
20389            ))
20390            .expect("next replacement worker consumes the second arrival");
20391        assert_eq!(completed.commands.len(), 1);
20392        assert_eq!(completed.commands[0]["type"], "complete_workflow");
20393        assert_eq!(
20394            decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20395                .expect("workflow result"),
20396            json!(["message-1", "message-2"])
20397        );
20398        assert_eq!(
20399            completed.message_stream_cursors,
20400            vec![json!({"stream_name": "orders", "through_position": 2})]
20401        );
20402        assert!(completed.message_stream_waits.is_empty());
20403
20404        let replay_history = vec![
20405            opened(1),
20406            first_applied,
20407            opened(2),
20408            applied(2, second_delivery),
20409        ];
20410        for _cold_worker_or_restart in 0..2 {
20411            let replayed = worker()
20412                .execute_workflow_task_decision(workflow_task(
20413                    "rust.message-stream",
20414                    replay_history.clone(),
20415                    DEFAULT_CODEC,
20416                ))
20417                .expect("cold worker replays each logical message exactly once");
20418            assert_eq!(replayed.commands.len(), 1);
20419            assert_eq!(
20420                decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20421                    .expect("replayed workflow result"),
20422                json!(["message-1", "message-2"])
20423            );
20424            assert_eq!(
20425                replayed.message_stream_cursors,
20426                vec![json!({"stream_name": "orders", "through_position": 2})]
20427            );
20428            assert!(replayed.message_stream_waits.is_empty());
20429        }
20430    }
20431
20432    #[test]
20433    fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20434        assert!(!worker_protocol_supports_message_streams("1.14"));
20435        assert!(worker_protocol_supports_message_streams("1.15"));
20436        assert!(worker_protocol_supports_message_streams("1.16"));
20437        assert!(worker_protocol_supports_message_streams(
20438            WORKER_PROTOCOL_VERSION
20439        ));
20440        assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20441    }
20442
20443    #[test]
20444    fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20445        let ctx = workflow_context(vec![
20446            history_event(
20447                "ConditionWaitOpened",
20448                json!({
20449                    "sequence": 1,
20450                    "condition_wait_id": "condition:1",
20451                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20452                    "condition_key": "signal:finish",
20453                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20454                }),
20455            ),
20456            history_event(
20457                "ConditionWaitSatisfied",
20458                json!({
20459                    "sequence": 1,
20460                    "condition_wait_id": "condition:1",
20461                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20462                    "condition_key": "signal:finish",
20463                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20464                }),
20465            ),
20466            history_event(
20467                "SignalReceived",
20468                json!({"signal_name": "finish", "arguments": []}),
20469            ),
20470        ]);
20471        let mut signal = Box::pin(ctx.wait_signal("finish"));
20472        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20473
20474        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20475            signal.as_mut().poll(&mut task_context)
20476        else {
20477            panic!("condition history must not resolve as a typed signal wait");
20478        };
20479        assert_eq!(failure.reason, "recorded_command_mismatch");
20480        assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20481    }
20482
20483    #[test]
20484    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20485        let signal_then_timer = vec![
20486            history_event(
20487                "SignalWaitOpened",
20488                json!({"sequence": 1, "signal_name": "go"}),
20489            ),
20490            history_event(
20491                "SignalApplied",
20492                json!({
20493                    "sequence": 1,
20494                    "signal_name": "go",
20495                    "value": fixture_envelope(json!(["now"])),
20496                }),
20497            ),
20498            history_event(
20499                "TimerScheduled",
20500                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20501            ),
20502            history_event(
20503                "TimerFired",
20504                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20505            ),
20506        ];
20507
20508        let ctx = workflow_context(signal_then_timer.clone());
20509        let mut signal = Box::pin(ctx.wait_signal("go"));
20510        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20511        assert!(matches!(
20512            signal.as_mut().poll(&mut task_context),
20513            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20514        ));
20515        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20516        assert!(matches!(
20517            timer.as_mut().poll(&mut task_context),
20518            Poll::Ready(Ok(()))
20519        ));
20520        ctx.ensure_history_consumed()
20521            .expect("signal and timer history consumed in order");
20522
20523        let reordered = workflow_context(signal_then_timer);
20524        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20525        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20526            timer_first.as_mut().poll(&mut task_context)
20527        else {
20528            panic!("timer cannot consume signal-wait-first history");
20529        };
20530        assert_eq!(failure.reason, "recorded_command_mismatch");
20531        assert_eq!(failure.sequence, Some(1));
20532        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20533
20534        let timer_then_signal = vec![
20535            history_event(
20536                "TimerScheduled",
20537                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20538            ),
20539            history_event(
20540                "TimerFired",
20541                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20542            ),
20543            history_event(
20544                "SignalWaitOpened",
20545                json!({"sequence": 2, "signal_name": "go"}),
20546            ),
20547            history_event(
20548                "SignalApplied",
20549                json!({
20550                    "sequence": 2,
20551                    "signal_name": "go",
20552                    "value": fixture_envelope(json!([])),
20553                }),
20554            ),
20555        ];
20556        let reordered = workflow_context(timer_then_signal);
20557        let mut signal_first = Box::pin(reordered.wait_signal("go"));
20558        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20559            signal_first.as_mut().poll(&mut task_context)
20560        else {
20561            panic!("signal wait cannot consume timer-first history");
20562        };
20563        assert_eq!(failure.reason, "recorded_command_mismatch");
20564        assert_eq!(failure.sequence, Some(1));
20565        assert_eq!(failure.expected.as_deref(), Some("timer"));
20566    }
20567
20568    #[test]
20569    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20570        let duplicate_timer = WorkflowState::new(
20571            vec![
20572                history_event(
20573                    "TimerScheduled",
20574                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20575                ),
20576                history_event(
20577                    "TimerScheduled",
20578                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20579                ),
20580            ],
20581            "rust-workers".to_string(),
20582            DEFAULT_CODEC.to_string(),
20583            None,
20584        )
20585        .expect_err("one workflow sequence cannot schedule two timers");
20586        assert!(matches!(
20587            duplicate_timer,
20588            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20589                if reason == "timer_schedule_missing_or_duplicate"
20590        ));
20591
20592        let colliding_kinds = WorkflowState::new(
20593            vec![
20594                history_event(
20595                    "TimerScheduled",
20596                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20597                ),
20598                history_event(
20599                    "ActivityCompleted",
20600                    json!({"sequence": 1, "activity_type": "same-sequence"}),
20601                ),
20602            ],
20603            "rust-workers".to_string(),
20604            DEFAULT_CODEC.to_string(),
20605            None,
20606        )
20607        .expect_err("one workflow sequence cannot identify two command kinds");
20608        assert!(matches!(
20609            colliding_kinds,
20610            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20611                if reason == "durable_command_sequence_collision"
20612        ));
20613
20614        let duplicate_signal_wait = WorkflowState::new(
20615            vec![
20616                history_event(
20617                    "SignalWaitOpened",
20618                    json!({"sequence": 1, "signal_name": "go"}),
20619                ),
20620                history_event(
20621                    "SignalWaitOpened",
20622                    json!({"sequence": 1, "signal_name": "go"}),
20623                ),
20624            ],
20625            "rust-workers".to_string(),
20626            DEFAULT_CODEC.to_string(),
20627            None,
20628        )
20629        .expect_err("one workflow sequence cannot open two signal waits");
20630        assert!(matches!(
20631            duplicate_signal_wait,
20632            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20633                if reason == "signal_wait_open_missing_or_duplicate"
20634        ));
20635    }
20636
20637    #[test]
20638    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20639        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20640            .expect("side-effect result");
20641        let ctx = workflow_context(vec![history_event(
20642            "SideEffectRecorded",
20643            json!({"sequence": 99, "result": result}),
20644        )]);
20645
20646        let replayed: Value = ctx
20647            .side_effect(|| panic!("recorded side effect must not run"))
20648            .expect("positive global workflow sequence is valid");
20649        assert_eq!(replayed, json!({"captured": true}));
20650        ctx.ensure_history_consumed().expect("history consumed");
20651    }
20652
20653    #[test]
20654    fn workflow_history_rejects_zero_and_descending_command_sequences() {
20655        let result =
20656            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20657        let zero = WorkflowState::new(
20658            vec![history_event(
20659                "SideEffectRecorded",
20660                json!({"sequence": 0, "result": result.clone()}),
20661            )],
20662            "rust-workers".to_string(),
20663            DEFAULT_CODEC.to_string(),
20664            None,
20665        )
20666        .expect_err("durable command sequences must be positive");
20667        assert!(matches!(
20668            zero,
20669            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20670                if reason == "durable_command_sequence_invalid"
20671        ));
20672
20673        let descending = WorkflowState::new(
20674            vec![
20675                history_event(
20676                    "SideEffectRecorded",
20677                    json!({"sequence": 3, "result": result}),
20678                ),
20679                history_event(
20680                    "VersionMarkerRecorded",
20681                    json!({
20682                        "sequence": 2,
20683                        "change_id": "descending-marker",
20684                        "version": 1,
20685                        "min_supported": 1,
20686                        "max_supported": 1,
20687                    }),
20688                ),
20689            ],
20690            "rust-workers".to_string(),
20691            DEFAULT_CODEC.to_string(),
20692            None,
20693        )
20694        .expect_err("new durable commands must remain strictly ordered");
20695        let Error::NonDeterministicReplay(failure) = descending else {
20696            panic!("expected typed replay failure");
20697        };
20698        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20699        assert_eq!(failure.sequence, Some(2));
20700        assert_eq!(
20701            failure.expected.as_deref(),
20702            Some("workflow sequence greater than 3")
20703        );
20704        assert_eq!(failure.actual.as_deref(), Some("2"));
20705    }
20706
20707    #[test]
20708    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20709        fn worker() -> Worker {
20710            let client = Client::new("http://127.0.0.1:8080").expect("client");
20711            let mut worker = Worker::new(client, "rust-workers");
20712            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20713                ctx.wait_signal("finish").await?;
20714                let marker: String =
20715                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20716                assert_eq!(marker, "after-finish");
20717                Ok(json!("finished"))
20718            });
20719            worker
20720        }
20721
20722        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20723            .expect("side-effect result");
20724        let task = workflow_task(
20725            "rust.finish-after-gaps",
20726            vec![
20727                history_event(
20728                    "SignalWaitOpened",
20729                    json!({"sequence": 1, "signal_name": "finish"}),
20730                ),
20731                history_event(
20732                    "SignalReceived",
20733                    json!({
20734                        "signal_id": "increment-3",
20735                        "signal_name": "increment",
20736                        "workflow_sequence": 2,
20737                        "payload_codec": DEFAULT_CODEC,
20738                        "arguments": fixture_envelope(json!([3])),
20739                    }),
20740                ),
20741                history_event(
20742                    "SignalReceived",
20743                    json!({
20744                        "signal_id": "increment-5",
20745                        "signal_name": "increment",
20746                        "workflow_sequence": 3,
20747                        "payload_codec": DEFAULT_CODEC,
20748                        "arguments": fixture_envelope(json!([5])),
20749                    }),
20750                ),
20751                history_event(
20752                    "SignalReceived",
20753                    json!({
20754                        "signal_id": "finish",
20755                        "signal_name": "finish",
20756                        "workflow_sequence": 4,
20757                        "payload_codec": DEFAULT_CODEC,
20758                        "arguments": fixture_envelope(json!([])),
20759                    }),
20760                ),
20761                history_event(
20762                    "SignalApplied",
20763                    json!({
20764                        "sequence": 1,
20765                        "signal_id": "finish",
20766                        "signal_name": "finish",
20767                        "payload_codec": DEFAULT_CODEC,
20768                        "value": fixture_envelope(json!([])),
20769                    }),
20770                ),
20771                history_event(
20772                    "SideEffectRecorded",
20773                    json!({"sequence": 5, "result": marker}),
20774                ),
20775            ],
20776            DEFAULT_CODEC,
20777        );
20778
20779        for _original_or_cold_worker in 0..2 {
20780            let commands = worker()
20781                .execute_workflow_task(task.clone())
20782                .expect("signal gaps preserve deterministic replay");
20783            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20784            assert_eq!(commands[0]["type"], "complete_workflow");
20785            assert_eq!(
20786                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20787                json!("finished")
20788            );
20789        }
20790    }
20791
20792    #[test]
20793    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20794        let ctx = workflow_context(Vec::new());
20795        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20796        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20797        assert!(matches!(
20798            sleep.as_mut().poll(&mut task_context),
20799            Poll::Ready(Err(Error::TimerDurationOverflow))
20800        ));
20801        assert!(ctx.take_commands().expect("commands").is_empty());
20802    }
20803
20804    #[test]
20805    fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20806        let entries = AvroValue::Map(BTreeMap::from([
20807            ("text".to_string(), AvroValue::String("same".to_string())),
20808            (
20809                "nested".to_string(),
20810                AvroValue::Map(BTreeMap::from([
20811                    ("beta".to_string(), AvroValue::Long(2)),
20812                    ("alpha".to_string(), AvroValue::Long(1)),
20813                ])),
20814            ),
20815            ("long".to_string(), AvroValue::Long(7)),
20816            ("double".to_string(), AvroValue::Double(7.0)),
20817            ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20818        ]));
20819        let ctx = workflow_context(Vec::new());
20820        ctx.upsert_memo(entries.clone()).expect("valid memo update");
20821        let commands = ctx.take_commands().expect("commands");
20822
20823        assert_eq!(commands.len(), 1);
20824        assert_eq!(commands[0]["type"], "upsert_memo");
20825        let server_entries = json!({
20826            "codec": "avro",
20827            "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20828        });
20829        assert_eq!(
20830            commands[0]["entries"]
20831                .as_object()
20832                .expect("entries envelope")
20833                .keys()
20834                .collect::<Vec<_>>(),
20835            vec!["blob", "codec"]
20836        );
20837        assert_eq!(commands[0]["entries"], server_entries);
20838        let wire_entries =
20839            decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20840        assert_eq!(wire_entries, entries);
20841
20842        let history = vec![history_event(
20843            "MemoUpserted",
20844            json!({
20845                "sequence": 1,
20846                "entries": server_entries.clone(),
20847                "merged": server_entries,
20848            }),
20849        )];
20850        let replay = workflow_context(history.clone());
20851        replay
20852            .upsert_memo(entries.clone())
20853            .expect("matching replay identity");
20854        assert!(replay.take_commands().expect("replay commands").is_empty());
20855
20856        let changed_types = AvroValue::Map(BTreeMap::from([
20857            ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20858            (
20859                "nested".to_string(),
20860                AvroValue::Map(BTreeMap::from([
20861                    ("alpha".to_string(), AvroValue::Long(1)),
20862                    ("beta".to_string(), AvroValue::Long(2)),
20863                ])),
20864            ),
20865            ("long".to_string(), AvroValue::Double(7.0)),
20866            ("double".to_string(), AvroValue::Long(7)),
20867            ("binary".to_string(), AvroValue::String("same".to_string())),
20868        ]));
20869        let error = workflow_context(history)
20870            .upsert_memo(changed_types)
20871            .expect_err("memo replay identity must preserve Avro value types");
20872        assert!(matches!(
20873            error,
20874            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20875        ));
20876    }
20877
20878    #[test]
20879    fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20880        let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20881            .expect("memo envelope");
20882        let replay = workflow_context(vec![history_event(
20883            "MemoUpserted",
20884            json!({
20885                "sequence": 1,
20886                "entries": original.clone(),
20887                "merged": original
20888            }),
20889        )]);
20890        let error = replay
20891            .upsert_memo(json!({"stage": "changed"}))
20892            .expect_err("changed memo update must fail replay");
20893        assert!(matches!(
20894            error,
20895            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20896        ));
20897
20898        let invalid = workflow_context(Vec::new())
20899            .upsert_memo(
20900                json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20901            )
20902            .expect_err("oversized key");
20903        assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20904    }
20905
20906    #[test]
20907    fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20908        let negative_zero = AvroValue::Map(BTreeMap::from([(
20909            "reading".to_string(),
20910            AvroValue::Double(-0.0),
20911        )]));
20912        let negative_zero_envelope =
20913            encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20914        let history = vec![history_event(
20915            "MemoUpserted",
20916            json!({
20917                "sequence": 1,
20918                "entries": negative_zero_envelope.clone(),
20919                "merged": negative_zero_envelope,
20920            }),
20921        )];
20922
20923        workflow_context(history.clone())
20924            .upsert_memo(negative_zero)
20925            .expect("matching negative-zero history identity");
20926
20927        let error = workflow_context(history)
20928            .upsert_memo(AvroValue::Map(BTreeMap::from([(
20929                "reading".to_string(),
20930                AvroValue::Double(0.0),
20931            )])))
20932            .expect_err("positive zero must not consume negative-zero memo history");
20933        assert!(matches!(
20934            error,
20935            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20936        ));
20937    }
20938
20939    #[test]
20940    fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20941        let supported = json!({
20942            "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20943            "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20944        });
20945        assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20946        assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20947            "workflow_memo_updates": {"supported": false},
20948            "supported_workflow_task_commands": ["upsert_memo"]
20949        }))));
20950        assert!(commands_use_workflow_memo_updates(&[json!({
20951            "type": "upsert_memo",
20952            "entries": {"stage": "processing"}
20953        })]));
20954    }
20955
20956    #[test]
20957    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20958        let client = Client::new("http://127.0.0.1:8080").expect("client");
20959        let mut worker = Worker::new(client, "rust-workers");
20960        worker.register_workflow("rust.timer", |ctx, _input| async move {
20961            ctx.sleep(Duration::from_secs(5)).await?;
20962            ctx.activity("after-timer", json!([])).await
20963        });
20964
20965        let task = |history_events| WorkflowTask {
20966            task_id: "wft-rust-timer-1".to_string(),
20967            workflow_command_id: None,
20968            workflow_id: Some("wf-rust-timer".to_string()),
20969            run_id: Some("run-rust-timer".to_string()),
20970            workflow_type: "rust.timer".to_string(),
20971            cancel_requested: false,
20972            payload_codec: DEFAULT_CODEC.to_string(),
20973            arguments: Some(
20974                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20975            ),
20976            history_events,
20977            total_history_events: None,
20978            history_size_bytes: None,
20979            continue_as_new_recommended: None,
20980            history_budget_pressure: None,
20981            next_history_page_token: None,
20982            workflow_task_attempt: 1,
20983            workflow_signal_id: None,
20984            signal_name: None,
20985            signal_arguments: None,
20986            workflow_update_id: None,
20987            update_name: None,
20988            lease_owner: Some("rust-worker".to_string()),
20989        };
20990
20991        let initial = worker
20992            .execute_workflow_task(task(Vec::new()))
20993            .expect("initial timer task");
20994        assert_eq!(
20995            initial,
20996            vec![json!({"type": "start_timer", "delay_seconds": 5})]
20997        );
20998
20999        let activity_result =
21000            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
21001        let replayed = worker
21002            .execute_workflow_task(task(vec![
21003                history_event(
21004                    "TimerScheduled",
21005                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21006                ),
21007                history_event(
21008                    "TimerFired",
21009                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21010                ),
21011                history_event(
21012                    "ActivityCompleted",
21013                    json!({
21014                        "sequence": 2,
21015                        "activity_type": "after-timer",
21016                        "payload_codec": DEFAULT_CODEC,
21017                        "result": activity_result,
21018                    }),
21019                ),
21020            ]))
21021            .expect("replayed workflow task");
21022        assert_eq!(replayed.len(), 1);
21023        assert_eq!(replayed[0]["type"], "complete_workflow");
21024        assert_eq!(
21025            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
21026            json!("done")
21027        );
21028    }
21029
21030    #[test]
21031    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
21032        let client = Client::new("http://127.0.0.1:8080").expect("client");
21033        let mut worker = Worker::new(client, "rust-workers");
21034        worker.register_workflow("rust.continue", |ctx, _input| async move {
21035            ctx.continue_as_new_with_options(
21036                ContinueAsNewOptions::new()
21037                    .workflow_type("rust.next")
21038                    .task_queue("next-workers"),
21039                json!([2, {"cursor": "next"}]),
21040            )
21041        });
21042
21043        let commands = worker
21044            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
21045            .expect("continue-as-new command");
21046
21047        assert_eq!(commands.len(), 1);
21048        assert_eq!(commands[0]["type"], "continue_as_new");
21049        assert_eq!(commands[0]["workflow_type"], "rust.next");
21050        assert_eq!(commands[0]["queue"], "next-workers");
21051        assert_eq!(
21052            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
21053                .expect("continue-as-new arguments"),
21054            json!([2, {"cursor": "next"}])
21055        );
21056    }
21057
21058    #[test]
21059    fn continue_as_new_preserves_typed_arguments() {
21060        let client = Client::new("http://127.0.0.1:8080").expect("client");
21061        let mut worker = Worker::new(client, "rust-workers");
21062        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
21063            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
21064            unreachable!("continue-as-new returns a control-flow error")
21065        });
21066
21067        let commands = worker
21068            .execute_workflow_task(workflow_task(
21069                "rust.typed-continue",
21070                Vec::new(),
21071                DEFAULT_CODEC,
21072            ))
21073            .expect("typed continue-as-new command");
21074
21075        assert_eq!(commands[0]["type"], "continue_as_new");
21076        assert_eq!(
21077            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
21078                .expect("typed continue arguments"),
21079            AvroValue::Array(vec![typed_fidelity_probe()])
21080        );
21081    }
21082
21083    #[test]
21084    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
21085        let client = Client::new("http://127.0.0.1:8080").expect("client");
21086        let mut worker = Worker::new(client, "rust-workers");
21087        worker.register_workflow("rust.continue", |ctx, _input| async move {
21088            ctx.continue_as_new(json!([2]))
21089        });
21090        let task = workflow_task(
21091            "rust.continue",
21092            vec![history_event(
21093                "WorkflowContinuedAsNew",
21094                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
21095            )],
21096            DEFAULT_CODEC,
21097        );
21098
21099        for _worker_restart_or_redelivery in 0..2 {
21100            let commands = worker
21101                .execute_workflow_task(task.clone())
21102                .expect("recorded transition replays");
21103            assert!(
21104                commands.is_empty(),
21105                "replay must not emit another successor"
21106            );
21107        }
21108    }
21109
21110    #[test]
21111    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
21112        let ctx = workflow_context(Vec::new());
21113        let error = ctx
21114            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
21115            .expect_err("blank queue must be rejected");
21116
21117        let Error::InvalidContinueAsNewOptions(error) = error else {
21118            panic!("expected typed continue-as-new validation error");
21119        };
21120        assert_eq!(error.field, "task_queue");
21121        assert!(ctx.take_commands().expect("commands").is_empty());
21122    }
21123
21124    #[test]
21125    fn workflow_context_exposes_server_history_budget() {
21126        let client = Client::new("http://127.0.0.1:8080").expect("client");
21127        let mut worker = Worker::new(client, "rust-workers");
21128        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
21129            let budget = ctx.history_budget()?;
21130            Ok(json!({
21131                "events": budget.event_count,
21132                "bytes": budget.size_bytes,
21133                "recommended": budget.continue_as_new_recommended,
21134                "pressure": budget.pressure,
21135            }))
21136        });
21137        let task: WorkflowTask = serde_json::from_value(json!({
21138            "task_id": "task-history-budget",
21139            "workflow_type": "rust.history-budget",
21140            "payload_codec": DEFAULT_CODEC,
21141            "history_events": [],
21142            "total_history_events": 480,
21143            "history_size_bytes": 1_048_576,
21144            "continue_as_new_recommended": true,
21145            "history_budget_pressure": "continue_as_new_recommended",
21146        }))
21147        .expect("published workflow task");
21148
21149        let commands = worker
21150            .execute_workflow_task(task)
21151            .expect("history-budget workflow");
21152        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
21153        assert_eq!(result["events"], 480);
21154        assert_eq!(result["bytes"], 1_048_576);
21155        assert_eq!(result["recommended"], true);
21156        assert_eq!(result["pressure"], "continue_as_new_recommended");
21157    }
21158
21159    #[test]
21160    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
21161        let client = Client::new("http://127.0.0.1:8080").expect("client");
21162        let mut worker = Worker::new(client, "rust-workers");
21163        worker.register_workflow("rust.failing", |_ctx, _input| async move {
21164            Err(Error::Codec("rust_conformance_failure".to_string()))
21165        });
21166        let task = WorkflowTask {
21167            task_id: "wft-rust-failing-1".to_string(),
21168            workflow_command_id: None,
21169            workflow_id: Some("wf-rust-failing".to_string()),
21170            run_id: Some("run-rust-failing".to_string()),
21171            workflow_type: "rust.failing".to_string(),
21172            cancel_requested: false,
21173            payload_codec: DEFAULT_CODEC.to_string(),
21174            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21175            history_events: Vec::new(),
21176            total_history_events: Some(0),
21177            history_size_bytes: None,
21178            continue_as_new_recommended: None,
21179            history_budget_pressure: None,
21180            next_history_page_token: None,
21181            workflow_task_attempt: 1,
21182            workflow_signal_id: None,
21183            signal_name: None,
21184            signal_arguments: None,
21185            workflow_update_id: None,
21186            update_name: None,
21187            lease_owner: Some("rust-worker".to_string()),
21188        };
21189
21190        let commands = worker
21191            .execute_workflow_task(task)
21192            .expect("handler failure becomes a workflow command");
21193
21194        assert_eq!(commands.len(), 1);
21195        assert_eq!(commands[0]["type"], "fail_workflow");
21196        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
21197        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
21198        assert_eq!(commands[0]["non_retryable"], false);
21199        assert_eq!(
21200            commands[0]["message"],
21201            "codec error: rust_conformance_failure"
21202        );
21203        assert_eq!(
21204            commands[0]["exception"]["message"],
21205            "codec error: rust_conformance_failure"
21206        );
21207    }
21208
21209    #[test]
21210    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
21211        let client = Client::new("http://127.0.0.1:8080").expect("client");
21212        let mut worker = Worker::new(client, "rust-workers");
21213        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
21214            let _: String = ctx.side_effect(|| "captured".to_string())?;
21215            Err(Error::WorkerLoop("application failure".to_string()))
21216        });
21217
21218        let commands = worker
21219            .execute_workflow_task(workflow_task(
21220                "rust.failing-after-side-effect",
21221                Vec::new(),
21222                DEFAULT_CODEC,
21223            ))
21224            .expect("ordinary failure remains a workflow decision");
21225
21226        assert_eq!(commands.len(), 2);
21227        assert_eq!(commands[0]["type"], "record_side_effect");
21228        assert_eq!(commands[1]["type"], "fail_workflow");
21229    }
21230
21231    #[test]
21232    fn uncaught_recorded_activity_failure_claims_only_its_persisted_boundary() {
21233        let client = Client::new("http://127.0.0.1:8080").expect("client");
21234        let mut worker = Worker::new(client, "rust-workers");
21235        worker.register_workflow("rust.redrive", |ctx, _input| async move {
21236            ctx.activity("greet", json!([])).await?;
21237            Ok(Value::Null)
21238        });
21239        worker.register_workflow("rust.redrive-translated", |ctx, _input| async move {
21240            match ctx.activity("greet", json!([])).await {
21241                Err(Error::ActivityFailed(_)) => Err(Error::WorkerLoop("translated".to_string())),
21242                result => result,
21243            }
21244        });
21245
21246        let failed = history_event(
21247            "ActivityFailed",
21248            json!({
21249                "sequence": 1,
21250                "activity_type": "greet",
21251                "activity_execution_id": "activity-1",
21252                "message": "failed"
21253            }),
21254        );
21255        let commands = worker
21256            .execute_workflow_task(workflow_task(
21257                "rust.redrive",
21258                vec![failed.clone()],
21259                DEFAULT_CODEC,
21260            ))
21261            .expect("recorded failure becomes a workflow command");
21262        assert_eq!(commands[0]["type"], "fail_workflow");
21263        assert_eq!(commands[0]["failed_step_sequence"], 1);
21264        assert_eq!(commands[0]["failed_activity_execution_id"], "activity-1");
21265
21266        let translated = worker
21267            .execute_workflow_task(workflow_task(
21268                "rust.redrive-translated",
21269                vec![failed],
21270                DEFAULT_CODEC,
21271            ))
21272            .expect("translated failure becomes a workflow command");
21273        assert_eq!(translated[0]["type"], "fail_workflow");
21274        assert!(translated[0].get("failed_step_sequence").is_none());
21275        assert!(translated[0].get("failed_activity_execution_id").is_none());
21276
21277        let without_identity = worker
21278            .execute_workflow_task(workflow_task(
21279                "rust.redrive",
21280                vec![history_event(
21281                    "ActivityFailed",
21282                    json!({"sequence": 1, "activity_type": "greet", "message": "failed"}),
21283                )],
21284                DEFAULT_CODEC,
21285            ))
21286            .expect("incomplete identity still fails the workflow");
21287        assert!(without_identity[0].get("failed_step_sequence").is_none());
21288
21289        let timed_out = worker
21290            .execute_workflow_task(workflow_task(
21291                "rust.redrive",
21292                vec![history_event(
21293                    "ActivityTimedOut",
21294                    json!({
21295                        "sequence": 1,
21296                        "activity_type": "greet",
21297                        "activity_execution_id": "activity-timeout",
21298                    }),
21299                )],
21300                DEFAULT_CODEC,
21301            ))
21302            .expect("timeout still fails the workflow");
21303        assert!(timed_out[0].get("failed_step_sequence").is_none());
21304    }
21305
21306    #[test]
21307    fn redriven_history_reuses_completed_prefix_and_reschedules_only_failed_step() {
21308        let client = Client::new("http://127.0.0.1:8080").expect("client");
21309        let mut worker = Worker::new(client, "rust-workers");
21310        worker.register_workflow("rust.redrive-successor", |ctx, _input| async move {
21311            let first = ctx.activity("first", json!([])).await?;
21312            let second = ctx.activity("second", json!([first.clone()])).await?;
21313            Ok(json!({"first": first, "second": second}))
21314        });
21315        let first = history_event(
21316            "ActivityCompleted",
21317            json!({
21318                "sequence": 1,
21319                "activity_type": "first",
21320                "result": encode_value_envelope(&json!("recorded"), DEFAULT_CODEC).expect("first result"),
21321                "payload_codec": DEFAULT_CODEC,
21322                "reused_from_run_id": "failed-run",
21323                "reused_activity_execution_id": "original-first",
21324            }),
21325        );
21326
21327        let retry = worker
21328            .execute_workflow_task(workflow_task(
21329                "rust.redrive-successor",
21330                vec![first.clone()],
21331                DEFAULT_CODEC,
21332            ))
21333            .expect("reused result replays before failed step");
21334        assert_eq!(retry.len(), 1);
21335        assert_eq!(retry[0]["type"], "schedule_activity");
21336        assert_eq!(retry[0]["activity_type"], "second");
21337        assert_eq!(
21338            decode_wire_value(&retry[0]["arguments"], DEFAULT_CODEC).expect("second arguments"),
21339            json!(["recorded"]),
21340        );
21341
21342        let second = history_event(
21343            "ActivityCompleted",
21344            json!({
21345                "sequence": 2,
21346                "activity_type": "second",
21347                "result": encode_value_envelope(&json!("retried"), DEFAULT_CODEC).expect("second result"),
21348                "payload_codec": DEFAULT_CODEC,
21349            }),
21350        );
21351        let completed = worker
21352            .execute_workflow_task(workflow_task(
21353                "rust.redrive-successor",
21354                vec![first, second],
21355                DEFAULT_CODEC,
21356            ))
21357            .expect("retried step completes the successor");
21358        assert_eq!(completed.len(), 1);
21359        assert_eq!(completed[0]["type"], "complete_workflow");
21360        assert_eq!(
21361            decode_wire_value(&completed[0]["result"], DEFAULT_CODEC).expect("workflow result"),
21362            json!({"first": "recorded", "second": "retried"}),
21363        );
21364    }
21365
21366    #[test]
21367    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
21368        let client = Client::new("http://127.0.0.1:8080").expect("client");
21369        let mut worker = Worker::new(client, "rust-workers");
21370        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
21371            Err(Error::WorkerLoop("application failure".to_string()))
21372        });
21373        let result =
21374            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
21375
21376        let error = worker
21377            .execute_workflow_task(workflow_task(
21378                "rust.removed-side-effect",
21379                vec![history_event(
21380                    "SideEffectRecorded",
21381                    json!({"sequence": 1, "result": result}),
21382                )],
21383                DEFAULT_CODEC,
21384            ))
21385            .expect_err("removed committed history must not become fail_workflow");
21386
21387        let Error::NonDeterministicReplay(failure) = error else {
21388            panic!("expected typed replay failure");
21389        };
21390        assert_eq!(failure.reason, "recorded_commands_unconsumed");
21391        assert_eq!(failure.sequence, Some(1));
21392        assert_eq!(failure.expected.as_deref(), Some("side effect"));
21393    }
21394
21395    #[test]
21396    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
21397        let client = Client::new("http://127.0.0.1:8080").expect("client");
21398        let mut worker = Worker::new(client, "rust-workers");
21399        worker.register_workflow(
21400            "rust.side-effect-before-marker-error",
21401            |ctx, _input| async move {
21402                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
21403                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
21404                ctx.get_version("restart-safe", 2, 2)?;
21405                Ok(Value::Null)
21406            },
21407        );
21408
21409        let error = worker
21410            .execute_workflow_task(workflow_task(
21411                "rust.side-effect-before-marker-error",
21412                vec![history_event(
21413                    "VersionMarkerRecorded",
21414                    json!({
21415                        "sequence": 1,
21416                        "change_id": "restart-safe",
21417                        "version": 1,
21418                        "min_supported": 1,
21419                        "max_supported": 1,
21420                    }),
21421                )],
21422                DEFAULT_CODEC,
21423            ))
21424            .expect_err("replay error must return no queued workflow commands");
21425
21426        let Error::NonDeterministicReplay(failure) = error else {
21427            panic!("expected typed replay failure");
21428        };
21429        assert_eq!(failure.reason, "version_marker_incompatible_range");
21430        assert_eq!(failure.sequence, Some(1));
21431    }
21432
21433    #[test]
21434    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
21435        let client = Client::new("http://127.0.0.1:8080").expect("client");
21436        let mut worker = Worker::new(client, "rust-workers");
21437        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
21438            ctx.sleep(Duration::from_secs(5)).await?;
21439            Ok(json!({"status": "timer fired"}))
21440        });
21441
21442        let task = WorkflowTask {
21443            task_id: "wft-rust-timer-pending".to_string(),
21444            workflow_command_id: None,
21445            workflow_id: Some("wf-rust-timer".to_string()),
21446            run_id: Some("run-rust-timer".to_string()),
21447            workflow_type: "rust.timer.pending".to_string(),
21448            cancel_requested: false,
21449            payload_codec: DEFAULT_CODEC.to_string(),
21450            arguments: Some(
21451                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21452            ),
21453            history_events: vec![history_event(
21454                "TimerScheduled",
21455                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21456            )],
21457            total_history_events: Some(1),
21458            history_size_bytes: None,
21459            continue_as_new_recommended: None,
21460            history_budget_pressure: None,
21461            next_history_page_token: None,
21462            workflow_task_attempt: 1,
21463            workflow_signal_id: None,
21464            signal_name: None,
21465            signal_arguments: None,
21466            workflow_update_id: None,
21467            update_name: None,
21468            lease_owner: Some("rust-worker".to_string()),
21469        };
21470
21471        for _redelivery_or_restart in 0..2 {
21472            let commands = worker
21473                .execute_workflow_task(task.clone())
21474                .expect("recorded timer remains pending");
21475            assert!(
21476                commands.is_empty(),
21477                "recorded timer must not be rescheduled"
21478            );
21479        }
21480    }
21481
21482    #[test]
21483    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21484        let client = Client::new("http://127.0.0.1:8080").expect("client");
21485        let mut worker = Worker::new(client, "rust-workers");
21486        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21487            Ok(json!({"status": "completed"}))
21488        });
21489        let task = WorkflowTask {
21490            task_id: "wft-rust-timer-removed".to_string(),
21491            workflow_command_id: None,
21492            workflow_id: Some("wf-rust-timer".to_string()),
21493            run_id: Some("run-rust-timer".to_string()),
21494            workflow_type: "rust.timer.removed".to_string(),
21495            cancel_requested: false,
21496            payload_codec: DEFAULT_CODEC.to_string(),
21497            arguments: Some(
21498                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21499            ),
21500            history_events: vec![
21501                history_event(
21502                    "TimerScheduled",
21503                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21504                ),
21505                history_event(
21506                    "TimerFired",
21507                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21508                ),
21509            ],
21510            total_history_events: Some(2),
21511            history_size_bytes: None,
21512            continue_as_new_recommended: None,
21513            history_budget_pressure: None,
21514            next_history_page_token: None,
21515            workflow_task_attempt: 1,
21516            workflow_signal_id: None,
21517            signal_name: None,
21518            signal_arguments: None,
21519            workflow_update_id: None,
21520            update_name: None,
21521            lease_owner: Some("rust-worker".to_string()),
21522        };
21523
21524        let Error::NonDeterministicReplay(failure) = worker
21525            .execute_workflow_task(task)
21526            .expect_err("removed timer must fail replay")
21527        else {
21528            panic!("expected typed replay failure");
21529        };
21530        assert_eq!(failure.reason, "recorded_commands_unconsumed");
21531        assert_eq!(failure.sequence, Some(1));
21532    }
21533
21534    #[test]
21535    fn workflow_context_emits_explicit_child_workflow_contract() {
21536        let ctx = WorkflowContext {
21537            state: Arc::new(Mutex::new(
21538                WorkflowState::new_with_identity(
21539                    Vec::new(),
21540                    Some("wf-parent".to_string()),
21541                    Some("run-parent".to_string()),
21542                    "parent-workers".to_string(),
21543                    DEFAULT_CODEC.to_string(),
21544                    None,
21545                )
21546                .expect("workflow state"),
21547            )),
21548        };
21549        let options = ChildWorkflowOptions::new("python-workers")
21550            .parent_close_policy(ParentClosePolicy::RequestCancel)
21551            .retry_policy(ChildWorkflowRetryPolicy {
21552                max_attempts: Some(3),
21553                backoff_seconds: vec![1, 5],
21554                non_retryable_error_types: vec!["ValidationError".to_string()],
21555            })
21556            .execution_timeout_seconds(600)
21557            .run_timeout_seconds(120);
21558        let mut call = Box::pin(ctx.start_child_workflow(
21559            "python.fulfil-order",
21560            options,
21561            json!([{"order_id": "order-42"}]),
21562        ));
21563        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21564
21565        assert!(matches!(
21566            call.as_mut().poll(&mut task_context),
21567            Poll::Pending
21568        ));
21569        let commands = ctx.take_commands().expect("commands");
21570        assert_eq!(commands.len(), 1);
21571        let command = &commands[0];
21572        assert_eq!(command["type"], "start_child_workflow");
21573        assert_eq!(command["workflow_type"], "python.fulfil-order");
21574        assert_eq!(command["queue"], "python-workers");
21575        assert_eq!(command["parent_close_policy"], "request_cancel");
21576        assert_eq!(command["retry_policy"]["max_attempts"], 3);
21577        assert_eq!(command["execution_timeout_seconds"], 600);
21578        assert_eq!(command["run_timeout_seconds"], 120);
21579        assert_eq!(
21580            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21581            json!([{"order_id": "order-42"}])
21582        );
21583    }
21584
21585    fn child_parent_worker() -> Worker {
21586        let client = Client::new("http://127.0.0.1:8080").expect("client");
21587        let mut worker = Worker::new(client, "rust-parent-workers");
21588        worker.register_workflow("rust.parent", |ctx, _input| async move {
21589            let child = ctx
21590                .start_child_workflow(
21591                    "python.child",
21592                    ChildWorkflowOptions::new("python-child-workers")
21593                        .parent_close_policy(ParentClosePolicy::Terminate),
21594                    json!([{"codec_probe": [1, true, "rust"]}]),
21595                )
21596                .await?;
21597            Ok(json!({
21598                "parent_workflow_id": child.parent.workflow_id,
21599                "parent_run_id": child.parent.run_id,
21600                "child_workflow_id": child.child.workflow_id,
21601                "child_run_id": child.child.run_id,
21602                "child_workflow_type": child.child_workflow_type,
21603                "result": child.result,
21604            }))
21605        });
21606        worker
21607    }
21608
21609    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21610        WorkflowTask {
21611            task_id: "wft-child-parent".to_string(),
21612            workflow_command_id: None,
21613            workflow_id: Some("wf-parent".to_string()),
21614            run_id: Some("run-parent".to_string()),
21615            workflow_type: "rust.parent".to_string(),
21616            cancel_requested: false,
21617            payload_codec: DEFAULT_CODEC.to_string(),
21618            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21619            history_events: vec![
21620                HistoryEvent {
21621                    event_type: "ChildWorkflowScheduled".to_string(),
21622                    payload: json!({
21623                        "sequence": 1,
21624                        "child_call_id": "call-child",
21625                        "child_workflow_instance_id": "wf-child",
21626                        "child_workflow_run_id": "run-child",
21627                        "child_workflow_type": "python.child",
21628                    }),
21629                    raw: HashMap::new(),
21630                },
21631                HistoryEvent {
21632                    event_type: event_type.to_string(),
21633                    payload,
21634                    raw: HashMap::new(),
21635                },
21636            ],
21637            total_history_events: Some(2),
21638            history_size_bytes: None,
21639            continue_as_new_recommended: None,
21640            history_budget_pressure: None,
21641            next_history_page_token: None,
21642            workflow_task_attempt: 1,
21643            workflow_signal_id: None,
21644            signal_name: None,
21645            signal_arguments: None,
21646            workflow_update_id: None,
21647            update_name: None,
21648            lease_owner: Some("rust-worker".to_string()),
21649        }
21650    }
21651
21652    #[test]
21653    fn committed_child_result_replays_without_starting_a_duplicate() {
21654        let worker = child_parent_worker();
21655        let task = child_parent_task(
21656            "ChildRunCompleted",
21657            json!({
21658                "sequence": 1,
21659                "child_call_id": "call-child",
21660                "child_workflow_instance_id": "wf-child",
21661                "child_workflow_run_id": "run-child",
21662                "child_workflow_type": "python.child",
21663                "payload_codec": DEFAULT_CODEC,
21664                "result": fixture_envelope(json!({"from":"python","ok":true})),
21665            }),
21666        );
21667
21668        for _restart in 0..2 {
21669            let commands = worker
21670                .execute_workflow_task(task.clone())
21671                .expect("replayed parent task");
21672            assert_eq!(commands.len(), 1);
21673            assert_eq!(commands[0]["type"], "complete_workflow");
21674            assert!(!commands
21675                .iter()
21676                .any(|command| command["type"] == "start_child_workflow"));
21677            let output =
21678                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21679            assert_eq!(output["parent_workflow_id"], "wf-parent");
21680            assert_eq!(output["parent_run_id"], "run-parent");
21681            assert_eq!(output["child_workflow_id"], "wf-child");
21682            assert_eq!(output["child_run_id"], "run-child");
21683            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21684        }
21685    }
21686
21687    #[test]
21688    fn typed_child_arguments_and_results_survive_replay() {
21689        let client = Client::new("http://127.0.0.1:8080").expect("client");
21690        let mut worker = Worker::new(client, "rust-parent-workers");
21691        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21692            let child = ctx
21693                .start_child_workflow_avro_value(
21694                    "python.typed-child",
21695                    ChildWorkflowOptions::new("python-workers"),
21696                    AvroValue::Array(vec![typed_fidelity_probe()]),
21697                )
21698                .await?;
21699            Ok(child.result)
21700        });
21701
21702        let initial = worker
21703            .execute_workflow_task(workflow_task(
21704                "rust.typed-parent",
21705                Vec::new(),
21706                DEFAULT_CODEC,
21707            ))
21708            .expect("typed child start");
21709        assert_eq!(initial[0]["type"], "start_child_workflow");
21710        assert_eq!(
21711            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21712                .expect("typed child arguments"),
21713            AvroValue::Array(vec![typed_fidelity_probe()])
21714        );
21715
21716        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21717            .expect("typed child result");
21718        let task = workflow_task(
21719            "rust.typed-parent",
21720            vec![
21721                history_event(
21722                    "ChildWorkflowScheduled",
21723                    json!({
21724                        "sequence": 1,
21725                        "child_call_id": "call-typed",
21726                        "child_workflow_instance_id": "wf-child",
21727                        "child_workflow_run_id": "run-child",
21728                        "child_workflow_type": "python.typed-child",
21729                    }),
21730                ),
21731                history_event(
21732                    "ChildRunCompleted",
21733                    json!({
21734                        "sequence": 1,
21735                        "child_call_id": "call-typed",
21736                        "child_workflow_instance_id": "wf-child",
21737                        "child_workflow_run_id": "run-child",
21738                        "child_workflow_type": "python.typed-child",
21739                        "payload_codec": DEFAULT_CODEC,
21740                        "result": result,
21741                    }),
21742                ),
21743            ],
21744            DEFAULT_CODEC,
21745        );
21746
21747        let commands = worker
21748            .execute_workflow_task(task)
21749            .expect("typed child replay");
21750        assert_eq!(commands[0]["type"], "complete_workflow");
21751        assert_eq!(
21752            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21753                .expect("typed parent result"),
21754            typed_fidelity_probe()
21755        );
21756    }
21757
21758    #[test]
21759    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21760        let worker = child_parent_worker();
21761        let mut task = child_parent_task("unused", Value::Null);
21762        task.history_events.truncate(1);
21763        task.total_history_events = Some(1);
21764
21765        for _redelivery_or_restart in 0..2 {
21766            let commands = worker
21767                .execute_workflow_task(task.clone())
21768                .expect("recorded child remains pending");
21769            assert!(
21770                commands.is_empty(),
21771                "recorded pending child must not be started again"
21772            );
21773        }
21774    }
21775
21776    #[test]
21777    fn child_cancellation_becomes_stable_parent_failure_command() {
21778        let worker = child_parent_worker();
21779        let task = child_parent_task(
21780            "ChildRunCancelled",
21781            json!({
21782                "sequence": 1,
21783                "child_workflow_instance_id": "wf-child",
21784                "child_workflow_run_id": "run-child",
21785                "child_workflow_type": "python.child",
21786                "failure_id": "failure-child",
21787                "failure_category": "cancelled",
21788                "message": "cancelled by parent-close policy",
21789            }),
21790        );
21791
21792        let commands = worker
21793            .execute_workflow_task(task)
21794            .expect("parent settlement");
21795        assert_eq!(commands.len(), 1);
21796        assert_eq!(commands[0]["type"], "fail_workflow");
21797        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21798        assert_eq!(
21799            commands[0]["exception"]["properties"]["reason"],
21800            "cancelled"
21801        );
21802        assert_eq!(
21803            commands[0]["exception"]["properties"]["child_workflow_run_id"],
21804            "run-child"
21805        );
21806    }
21807
21808    #[test]
21809    fn workflow_can_handle_typed_child_failure() {
21810        let client = Client::new("http://127.0.0.1:8080").expect("client");
21811        let mut worker = Worker::new(client, "rust-parent-workers");
21812        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21813            match ctx
21814                .start_child_workflow(
21815                    "python.child",
21816                    ChildWorkflowOptions::new("python-child-workers"),
21817                    json!([]),
21818                )
21819                .await
21820            {
21821                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21822                    "reason": failure.reason,
21823                    "failure_id": failure.failure_id,
21824                    "exception_class": failure.exception_class,
21825                    "child_run_id": failure.child_workflow_run_id,
21826                })),
21827                Err(error) => Err(error),
21828                Ok(_) => Err(Error::WorkerLoop(
21829                    "child unexpectedly succeeded".to_string(),
21830                )),
21831            }
21832        });
21833        let mut task = child_parent_task(
21834            "ChildRunFailed",
21835            json!({
21836                "sequence": 1,
21837                "child_workflow_instance_id": "wf-child",
21838                "child_workflow_run_id": "run-child",
21839                "child_workflow_type": "python.child",
21840                "failure_id": "failure-child",
21841                "failure_category": "child_workflow",
21842                "message": "payment rejected",
21843                "exception": {
21844                    "type": "PaymentRejected",
21845                    "class": "payments.PaymentRejected",
21846                    "message": "payment rejected"
21847                }
21848            }),
21849        );
21850        task.workflow_type = "rust.handled-parent".to_string();
21851
21852        let commands = worker.execute_workflow_task(task).expect("handled failure");
21853        assert_eq!(commands[0]["type"], "complete_workflow");
21854        let output =
21855            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21856        assert_eq!(output["reason"], "child_workflow");
21857        assert_eq!(output["failure_id"], "failure-child");
21858        assert_eq!(output["exception_class"], "payments.PaymentRejected");
21859        assert_eq!(output["child_run_id"], "run-child");
21860    }
21861
21862    #[test]
21863    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21864        let client = Client::new("http://127.0.0.1:8080").expect("client");
21865        let mut worker = Worker::new(client, "rust-workers");
21866
21867        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21868            let signal = ctx.wait_signal("start").await?;
21869            let name = signal
21870                .first()
21871                .and_then(|value| value.as_str())
21872                .unwrap_or("world");
21873            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21874            Ok(json!({
21875                "greeting": greeting,
21876                "language": "rust"
21877            }))
21878        });
21879
21880        let signal_arguments =
21881            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21882        let task = WorkflowTask {
21883            task_id: "wft-rust-signal-1".to_string(),
21884            workflow_command_id: None,
21885            workflow_id: Some("wf-rust-hello".to_string()),
21886            run_id: Some("run-rust-hello".to_string()),
21887            workflow_type: "rust.hello_workflow".to_string(),
21888            cancel_requested: false,
21889            payload_codec: DEFAULT_CODEC.to_string(),
21890            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21891            history_events: vec![HistoryEvent {
21892                event_type: "SignalReceived".to_string(),
21893                payload: json!({
21894                    "signal_id": "sig-rust-1",
21895                    "signal_name": "start"
21896                }),
21897                raw: HashMap::new(),
21898            }],
21899            total_history_events: Some(1),
21900            history_size_bytes: None,
21901            continue_as_new_recommended: None,
21902            history_budget_pressure: None,
21903            next_history_page_token: None,
21904            workflow_task_attempt: 1,
21905            workflow_signal_id: Some("sig-rust-1".to_string()),
21906            signal_name: Some("start".to_string()),
21907            signal_arguments: Some(signal_arguments),
21908            workflow_update_id: None,
21909            update_name: None,
21910            lease_owner: Some("rust-worker".to_string()),
21911        };
21912
21913        let commands = worker.execute_workflow_task(task).expect("workflow task");
21914
21915        assert_eq!(commands.len(), 1);
21916        assert_eq!(commands[0]["type"], "schedule_activity");
21917        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21918        assert_eq!(
21919            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21920            json!(["Rust"])
21921        );
21922    }
21923
21924    #[test]
21925    fn workflow_task_appends_paginated_history_events() {
21926        let mut task = WorkflowTask {
21927            task_id: "wft-rust-pages-1".to_string(),
21928            workflow_command_id: None,
21929            workflow_id: Some("wf-rust-pages".to_string()),
21930            run_id: Some("run-rust-pages".to_string()),
21931            workflow_type: "rust.hello_workflow".to_string(),
21932            cancel_requested: false,
21933            payload_codec: DEFAULT_CODEC.to_string(),
21934            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21935            history_events: vec![HistoryEvent {
21936                event_type: "WorkflowStarted".to_string(),
21937                payload: json!({}),
21938                raw: HashMap::new(),
21939            }],
21940            total_history_events: Some(3),
21941            history_size_bytes: None,
21942            continue_as_new_recommended: None,
21943            history_budget_pressure: None,
21944            next_history_page_token: Some("MQ==".to_string()),
21945            workflow_task_attempt: 1,
21946            workflow_signal_id: None,
21947            signal_name: None,
21948            signal_arguments: None,
21949            workflow_update_id: None,
21950            update_name: None,
21951            lease_owner: Some("rust-worker".to_string()),
21952        };
21953
21954        task.append_history_page(WorkflowTaskHistoryPage {
21955            history_events: vec![
21956                HistoryEvent {
21957                    event_type: "SignalReceived".to_string(),
21958                    payload: json!({
21959                        "signal_id": "sig-rust-1",
21960                        "signal_name": "start",
21961                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21962                            .expect("signal arguments")
21963                    }),
21964                    raw: HashMap::new(),
21965                },
21966                HistoryEvent {
21967                    event_type: "MarkerRecorded".to_string(),
21968                    payload: json!({"sequence": 3}),
21969                    raw: HashMap::new(),
21970                },
21971            ],
21972            total_history_events: Some(3),
21973            next_history_page_token: None,
21974        });
21975
21976        assert_eq!(task.history_events.len(), 3);
21977        assert_eq!(task.total_history_events, Some(3));
21978        assert_eq!(task.next_history_page_token, None);
21979
21980        let signal = task
21981            .history_events
21982            .iter()
21983            .find(|event| event.event_type == "SignalReceived")
21984            .expect("signal event");
21985        assert_eq!(
21986            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21987            vec![AvroValue::String("Rust".to_string())]
21988        );
21989    }
21990
21991    #[tokio::test]
21992    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21993        let client = Client::new("http://127.0.0.1:8080").expect("client");
21994        let mut worker = Worker::new(client, "rust-workers");
21995        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21996        worker.register_query("counter", "current", |ctx, _args| async move {
21997            let mut count = 0_i64;
21998            for signal in ctx.signal_events() {
21999                let value = signal
22000                    .arguments
22001                    .first()
22002                    .and_then(Value::as_i64)
22003                    .unwrap_or_default();
22004                match signal.name.as_str() {
22005                    "increment" => count += value,
22006                    "set" => count = value,
22007                    _ => {}
22008                }
22009            }
22010            Ok(json!(count))
22011        });
22012
22013        let task = QueryTask {
22014            query_task_id: "query-rust-counter".to_string(),
22015            query_task_attempt: 1,
22016            lease_owner: Some("rust-worker".to_string()),
22017            workflow_id: Some("counter-1".to_string()),
22018            run_id: Some("run-counter-1".to_string()),
22019            workflow_type: "counter".to_string(),
22020            query_name: "current".to_string(),
22021            payload_codec: DEFAULT_CODEC.to_string(),
22022            workflow_arguments: Some(
22023                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
22024            ),
22025            query_arguments: Some(
22026                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
22027            ),
22028            history_events: vec![
22029                HistoryEvent {
22030                    event_type: "SignalReceived".to_string(),
22031                    payload: json!({
22032                        "signal_id": "php-signal-1",
22033                        "signal_name": "increment",
22034                        "workflow_sequence": 1,
22035                        "payload_codec": DEFAULT_CODEC,
22036                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
22037                    }),
22038                    raw: HashMap::new(),
22039                },
22040                HistoryEvent {
22041                    event_type: "SignalReceived".to_string(),
22042                    payload: json!({
22043                        "signal_id": "python-signal-2",
22044                        "signal_name": "increment",
22045                        "workflow_sequence": 2,
22046                        "payload_codec": DEFAULT_CODEC,
22047                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
22048                    }),
22049                    raw: HashMap::new(),
22050                },
22051                HistoryEvent {
22052                    event_type: "SignalReceived".to_string(),
22053                    payload: json!({
22054                        "signal_id": "rust-signal-3",
22055                        "signal_name": "set",
22056                        "workflow_sequence": 3,
22057                        "payload_codec": DEFAULT_CODEC,
22058                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
22059                    }),
22060                    raw: HashMap::new(),
22061                },
22062            ],
22063            history_export: None,
22064            run_status: Some("completed".to_string()),
22065        };
22066
22067        let result = worker.execute_query_task(task).await.expect("query result");
22068        assert_eq!(result.into_json().expect("query projection"), json!(0));
22069    }
22070
22071    #[tokio::test]
22072    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
22073        let worker = replay_counter_worker();
22074        let running_history = json!([
22075            {
22076                "type": "ActivityCompleted",
22077                "payload": {
22078                    "sequence": 1,
22079                    "activity_type": "load-counter",
22080                    "payload_codec": DEFAULT_CODEC,
22081                    "result": fixture_envelope(json!("loaded"))
22082                }
22083            },
22084            {
22085                "type": "SignalWaitOpened",
22086                "payload": {
22087                    "sequence": 3,
22088                    "signal_name": "increment"
22089                }
22090            },
22091            {
22092                "type": "SignalReceived",
22093                "payload": {
22094                    "signal_id": "signal-3",
22095                    "signal_name": "increment",
22096                    "workflow_sequence": 2,
22097                    "payload_codec": DEFAULT_CODEC,
22098                    "arguments": fixture_envelope(json!([3]))
22099                }
22100            },
22101            {
22102                "type": "SignalApplied",
22103                "payload": {
22104                    "sequence": 3,
22105                    "signal_id": "signal-3",
22106                    "signal_name": "increment",
22107                    "payload_codec": DEFAULT_CODEC,
22108                    "value": fixture_envelope(json!([3]))
22109                }
22110            }
22111        ]);
22112
22113        let running = worker
22114            .execute_query_task(replay_counter_query(
22115                "current",
22116                running_history.clone(),
22117                "running",
22118            ))
22119            .await
22120            .expect("running replay query");
22121        assert_eq!(
22122            running.clone().into_json().expect("query projection"),
22123            json!({"loaded": "loaded", "count": 3, "finished": false})
22124        );
22125
22126        let detached = worker
22127            .execute_query_task(replay_counter_query(
22128                "detached-mutation",
22129                running_history.clone(),
22130                "running",
22131            ))
22132            .await
22133            .expect("query mutates only its detached state clone");
22134        assert_eq!(detached.into_json().expect("query projection"), json!(999));
22135        let failed = worker
22136            .execute_query_task(replay_counter_query(
22137                "failed-mutation",
22138                running_history.clone(),
22139                "running",
22140            ))
22141            .await
22142            .expect_err("failed query");
22143        assert_eq!(failed.reason, "query_rejected");
22144        let unchanged = worker
22145            .execute_query_task(replay_counter_query("current", running_history, "running"))
22146            .await
22147            .expect("later query reconstructs unchanged state");
22148        assert_eq!(unchanged, running);
22149
22150        let restarted_worker = replay_counter_worker();
22151        let empty_arguments = fixture_envelope(json!([]));
22152        let loaded_result = fixture_envelope(json!("loaded"));
22153        let signal_three = fixture_blob(json!([3]));
22154        let signal_five = fixture_blob(json!([5]));
22155        let restarted_task: QueryTask = serde_json::from_value(json!({
22156            "query_task_id": "query-after-restart",
22157            "workflow_id": "counter-1",
22158            "run_id": "run-counter-1",
22159            "workflow_type": "replay-counter",
22160            "query_name": "current",
22161            "payload_codec": DEFAULT_CODEC,
22162            "workflow_arguments": empty_arguments.clone(),
22163            "query_arguments": empty_arguments,
22164            "history_events": [],
22165            "history_export": {
22166                "payloads": {"codec": DEFAULT_CODEC},
22167                "history_events": [
22168                    {
22169                        "type": "ActivityCompleted",
22170                        "payload": {
22171                            "sequence": 1,
22172                            "activity_type": "load-counter",
22173                            "payload_codec": DEFAULT_CODEC,
22174                            "result": null
22175                        }
22176                    },
22177                    {
22178                        "type": "SignalWaitOpened",
22179                        "payload": {
22180                            "sequence": 3,
22181                            "signal_name": "increment"
22182                        }
22183                    },
22184                    {
22185                        "type": "SignalReceived",
22186                        "payload": {
22187                            "signal_id": "signal-3",
22188                            "signal_name": "increment",
22189                            "workflow_sequence": 2
22190                        }
22191                    },
22192                    {
22193                        "type": "SignalApplied",
22194                        "payload": {
22195                            "sequence": 3,
22196                            "signal_id": "signal-3",
22197                            "signal_name": "increment"
22198                        }
22199                    },
22200                    {
22201                        "type": "SignalWaitOpened",
22202                        "payload": {
22203                            "sequence": 5,
22204                            "signal_name": "increment"
22205                        }
22206                    },
22207                    {
22208                        "type": "SignalReceived",
22209                        "payload": {
22210                            "signal_id": "signal-5",
22211                            "signal_name": "increment",
22212                            "workflow_sequence": 4
22213                        }
22214                    },
22215                    {
22216                        "type": "SignalApplied",
22217                        "payload": {
22218                            "sequence": 5,
22219                            "signal_id": "signal-5",
22220                            "signal_name": "increment"
22221                        }
22222                    }
22223                ],
22224                "activities": [{
22225                    "sequence": 1,
22226                    "activity_type": "load-counter",
22227                    "payload_codec": DEFAULT_CODEC,
22228                    "result": loaded_result
22229                }],
22230                "signals": [
22231                    {
22232                        "id": "signal-3",
22233                        "name": "increment",
22234                        "workflow_sequence": 2,
22235                        "payload_codec": DEFAULT_CODEC,
22236                        "arguments": signal_three
22237                    },
22238                    {
22239                        "id": "signal-5",
22240                        "name": "increment",
22241                        "workflow_sequence": 4,
22242                        "payload_codec": DEFAULT_CODEC,
22243                        "arguments": signal_five
22244                    }
22245                ]
22246            },
22247            "run_status": "completed"
22248        }))
22249        .expect("cold replay query task");
22250        let completed = restarted_worker
22251            .execute_query_task(restarted_task)
22252            .await
22253            .expect("completed cold replay query");
22254        assert_eq!(
22255            completed.into_json().expect("query projection"),
22256            json!({"loaded": "loaded", "count": 8, "finished": true})
22257        );
22258    }
22259
22260    #[tokio::test]
22261    async fn replayed_query_replay_failures_are_machine_readable() {
22262        let worker = replay_counter_worker();
22263        let task = replay_counter_query(
22264            "current",
22265            json!([{
22266                "type": "ActivityCompleted",
22267                "payload": {
22268                    "sequence": 1,
22269                    "payload_codec": DEFAULT_CODEC,
22270                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
22271                }
22272            }]),
22273            "running",
22274        );
22275        let failure = worker
22276            .execute_query_task(task)
22277            .await
22278            .expect_err("invalid replay history payload");
22279        assert_eq!(failure.reason, "query_payload_decode_failed");
22280        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
22281        assert!(failure.message.contains("invalid_payload_framing"));
22282    }
22283
22284    #[tokio::test]
22285    async fn query_task_restores_compact_history_from_export() {
22286        let client = Client::new("http://127.0.0.1:8080").expect("client");
22287        let mut worker = Worker::new(client, "rust-workers");
22288        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22289        worker.register_query("counter", "current", |ctx, _args| async move {
22290            Ok(json!(ctx.signals("increment")[0][0]))
22291        });
22292        let empty_arguments = fixture_envelope(json!([]));
22293        let exported_signal = fixture_blob(json!([9]));
22294        let task: QueryTask = serde_json::from_value(json!({
22295            "query_task_id": "query-export",
22296            "workflow_type": "counter",
22297            "query_name": "current",
22298            "payload_codec": DEFAULT_CODEC,
22299            "workflow_arguments": empty_arguments.clone(),
22300            "query_arguments": empty_arguments,
22301            "history_events": [],
22302            "history_export": {
22303                "payloads": {"codec": DEFAULT_CODEC},
22304                "history_events": [{
22305                    "type": "SignalReceived",
22306                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
22307                }],
22308                "signals": [{
22309                    "id": "signal-export",
22310                    "name": "increment",
22311                    "status": "applied",
22312                    "workflow_sequence": 1,
22313                    "payload_codec": DEFAULT_CODEC,
22314                    "arguments": exported_signal
22315                }]
22316            }
22317        }))
22318        .expect("query task");
22319
22320        let result = worker.execute_query_task(task).await.expect("query result");
22321        assert_eq!(result.into_json().expect("query projection"), json!(9));
22322    }
22323
22324    #[tokio::test]
22325    async fn query_task_failures_have_stable_reasons() {
22326        let client = Client::new("http://127.0.0.1:8080").expect("client");
22327        let mut worker = Worker::new(client, "rust-workers");
22328        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22329        worker.register_query(
22330            "counter",
22331            "current",
22332            |_ctx, _args| async move { Ok(json!(0)) },
22333        );
22334
22335        let base_task = QueryTask {
22336            query_task_id: "query-errors".to_string(),
22337            query_task_attempt: 1,
22338            lease_owner: None,
22339            workflow_id: Some("counter-errors".to_string()),
22340            run_id: Some("run-errors".to_string()),
22341            workflow_type: "counter".to_string(),
22342            query_name: "missing".to_string(),
22343            payload_codec: DEFAULT_CODEC.to_string(),
22344            workflow_arguments: Some(fixture_envelope(json!([]))),
22345            query_arguments: Some(fixture_envelope(json!([]))),
22346            history_events: Vec::new(),
22347            history_export: None,
22348            run_status: Some("running".to_string()),
22349        };
22350
22351        let unknown = worker
22352            .execute_query_task(base_task.clone())
22353            .await
22354            .expect_err("unknown query");
22355        assert_eq!(unknown.reason, "rejected_unknown_query");
22356
22357        let mut malformed = base_task;
22358        malformed.query_name = "current".to_string();
22359        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
22360        let malformed = worker
22361            .execute_query_task(malformed)
22362            .await
22363            .expect_err("malformed payload");
22364        assert_eq!(malformed.reason, "query_payload_decode_failed");
22365
22366        let client = Client::new("http://127.0.0.1:8080").expect("client");
22367        let mut unavailable_worker = Worker::new(client, "rust-workers");
22368        unavailable_worker
22369            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22370        let empty_arguments = fixture_envelope(json!([]));
22371        let unavailable_task: QueryTask = serde_json::from_value(json!({
22372            "query_task_id": "query-unavailable",
22373            "workflow_type": "counter",
22374            "query_name": "current",
22375            "payload_codec": DEFAULT_CODEC,
22376            "workflow_arguments": empty_arguments.clone(),
22377            "query_arguments": empty_arguments
22378        }))
22379        .expect("query task");
22380        let unavailable = unavailable_worker
22381            .execute_query_task(unavailable_task)
22382            .await
22383            .expect_err("query handler unavailable");
22384        assert_eq!(unavailable.reason, "query_handler_unavailable");
22385    }
22386
22387    #[tokio::test]
22388    async fn client_query_decodes_result_and_typed_failure() {
22389        let server = MockWorkerServer::start();
22390        let client = Client::builder(server.base_url())
22391            .timeout(Duration::from_secs(2))
22392            .build()
22393            .expect("client");
22394
22395        let result = client
22396            .query_workflow("counter-1", "current", json!([]))
22397            .await
22398            .expect("query result");
22399        assert_eq!(result, json!({"count": 8}));
22400
22401        let error = client
22402            .query_workflow("counter-1", "missing", json!([]))
22403            .await
22404            .expect_err("unknown query");
22405        let Error::QueryFailed(failure) = error else {
22406            panic!("expected typed query failure");
22407        };
22408        assert_eq!(failure.status, 404);
22409        assert_eq!(failure.reason, "rejected_unknown_query");
22410    }
22411
22412    #[tokio::test]
22413    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
22414        let server = MockWorkerServer::start();
22415        let client = Client::builder(server.base_url())
22416            .timeout(Duration::from_secs(2))
22417            .build()
22418            .expect("client");
22419        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
22420
22421        client
22422            .start_workflow(
22423                "typed.echo",
22424                "rust-workers",
22425                "typed-start",
22426                arguments.clone(),
22427            )
22428            .await
22429            .expect("typed workflow start");
22430        assert_eq!(
22431            decode_wire_avro_value(
22432                &server.request_body("/api/workflows")["input"],
22433                DEFAULT_CODEC,
22434            )
22435            .expect("typed start input"),
22436            arguments
22437        );
22438
22439        client
22440            .signal_workflow("typed-1", "changed", arguments.clone())
22441            .await
22442            .expect("typed signal");
22443        assert_eq!(
22444            decode_wire_avro_value(
22445                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
22446                DEFAULT_CODEC,
22447            )
22448            .expect("typed signal input"),
22449            arguments
22450        );
22451
22452        assert_eq!(
22453            client
22454                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22455                .await
22456                .expect("typed query"),
22457            typed_fidelity_probe()
22458        );
22459        assert_eq!(
22460            decode_wire_avro_value(
22461                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22462                DEFAULT_CODEC,
22463            )
22464            .expect("typed query input"),
22465            arguments
22466        );
22467
22468        assert_eq!(
22469            client
22470                .update_workflow_avro_value(
22471                    "typed-1",
22472                    "replace",
22473                    arguments.clone(),
22474                    Some("typed-request"),
22475                )
22476                .await
22477                .expect("typed update"),
22478            typed_fidelity_probe()
22479        );
22480        let update = server.request_body("/api/workflows/typed-1/update/replace");
22481        assert_eq!(update["request_id"], "typed-request");
22482        assert_eq!(
22483            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22484            arguments
22485        );
22486
22487        let handle = WorkflowHandle {
22488            client: client.clone(),
22489            workflow_id: "typed-1".to_string(),
22490            run_id: Some("run-typed-1".to_string()),
22491            workflow_type: "typed.echo".to_string(),
22492        };
22493        assert_eq!(
22494            handle
22495                .result_avro_value(WorkflowResultOptions::default())
22496                .await
22497                .expect("typed workflow result"),
22498            typed_fidelity_probe()
22499        );
22500
22501        client
22502            .complete_activity_task(
22503                "activity-typed",
22504                "attempt-typed",
22505                "rust-worker",
22506                typed_fidelity_probe(),
22507                DEFAULT_CODEC,
22508            )
22509            .await
22510            .expect("typed activity completion");
22511        assert_eq!(
22512            decode_wire_avro_value(
22513                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22514                    ["result"],
22515                DEFAULT_CODEC,
22516            )
22517            .expect("typed activity result"),
22518            typed_fidelity_probe()
22519        );
22520        client
22521            .fail_activity_task(
22522                "activity-typed",
22523                "attempt-typed",
22524                "rust-worker",
22525                "typed failure",
22526                true,
22527            )
22528            .await
22529            .expect("activity failure");
22530    }
22531
22532    #[tokio::test]
22533    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22534        let server = MockWorkerServer::start();
22535        let client = Client::builder(server.base_url())
22536            .timeout(Duration::from_secs(2))
22537            .build()
22538            .expect("client");
22539
22540        let options = WorkflowCommandOptions::new()
22541            .reason("cleanup requested")
22542            .request_id("cancel-17");
22543        let cancelled = client
22544            .cancel_workflow("wf-lifecycle", options)
22545            .await
22546            .expect("instance cancellation");
22547        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22548        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22549        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22550        assert_eq!(
22551            server.request_body("/api/workflows/wf-lifecycle/cancel"),
22552            json!({"reason":"cleanup requested","request_id":"cancel-17"})
22553        );
22554
22555        let terminated = client
22556            .terminate_workflow(
22557                "wf-lifecycle",
22558                WorkflowCommandOptions::new().reason("forced stop"),
22559            )
22560            .await
22561            .expect("instance termination");
22562        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22563        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22564
22565        client
22566            .cancel_workflow_run(
22567                "wf-lifecycle",
22568                "run-current",
22569                WorkflowCommandOptions::default(),
22570            )
22571            .await
22572            .expect("selected run cancellation");
22573        client
22574            .terminate_workflow_run(
22575                "wf-lifecycle",
22576                "run-current",
22577                WorkflowCommandOptions::default(),
22578            )
22579            .await
22580            .expect("selected run termination");
22581
22582        for (command, error) in [
22583            (
22584                WorkflowCommandKind::Cancel,
22585                client
22586                    .cancel_workflow_run(
22587                        "wf-lifecycle",
22588                        "run-stale",
22589                        WorkflowCommandOptions::default(),
22590                    )
22591                    .await
22592                    .expect_err("stale cancellation must be rejected"),
22593            ),
22594            (
22595                WorkflowCommandKind::Terminate,
22596                client
22597                    .terminate_workflow_run(
22598                        "wf-lifecycle",
22599                        "run-stale",
22600                        WorkflowCommandOptions::default(),
22601                    )
22602                    .await
22603                    .expect_err("stale termination must be rejected"),
22604            ),
22605        ] {
22606            let Error::WorkflowCommandRejected(rejection) = error else {
22607                panic!("expected typed command rejection");
22608            };
22609            assert_eq!(rejection.command, command);
22610            assert_eq!(rejection.status, 409);
22611            assert_eq!(rejection.reason, "historical_run_command_rejected");
22612            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22613            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22614        }
22615    }
22616
22617    #[tokio::test]
22618    async fn redrive_targets_a_failed_run_and_returns_successor_identity() {
22619        let server = MockWorkerServer::start();
22620        let client = Client::builder(server.base_url())
22621            .timeout(Duration::from_secs(2))
22622            .build()
22623            .expect("client");
22624
22625        let result = client
22626            .redrive_workflow_run("wf-lifecycle", "run-failed", Some("retry-1"))
22627            .await
22628            .expect("redrive accepted");
22629        assert_eq!(result.workflow_id, "wf-lifecycle");
22630        assert_eq!(result.source_run_id, "run-failed");
22631        assert_eq!(result.run_id, "run-successor");
22632        assert_eq!(result.resume_step_sequence, 2);
22633        assert_eq!(
22634            server.request_body("/api/workflows/wf-lifecycle/runs/run-failed/redrive"),
22635            json!({"request_id":"retry-1"})
22636        );
22637
22638        let repeated = client
22639            .redrive_workflow_run("wf-lifecycle", "run-failed-existing", Some("retry-1"))
22640            .await
22641            .expect("idempotent redrive response accepted");
22642        assert_eq!(repeated.run_id, "run-successor");
22643
22644        let error = client
22645            .redrive_workflow_run("wf-lifecycle", "run-completed", None)
22646            .await
22647            .expect_err("completed run must be rejected");
22648        let Error::Http { status, body } = error else {
22649            panic!("expected HTTP redrive rejection");
22650        };
22651        assert_eq!(status.as_u16(), 409);
22652        assert!(body.contains("run_not_failed"));
22653        assert_eq!(
22654            server.request_body("/api/workflows/wf-lifecycle/runs/run-completed/redrive"),
22655            json!({})
22656        );
22657
22658        let handle = WorkflowHandle {
22659            client: client.clone(),
22660            workflow_id: "wf-lifecycle".to_string(),
22661            run_id: Some("run-failed".to_string()),
22662            workflow_type: "test".to_string(),
22663        };
22664        assert_eq!(
22665            handle.redrive(None).await.expect("handle redrive").run_id,
22666            "run-successor"
22667        );
22668        let missing_run = WorkflowHandle {
22669            run_id: None,
22670            ..handle
22671        };
22672        assert!(matches!(
22673            missing_run.redrive(None).await,
22674            Err(Error::Codec(_))
22675        ));
22676    }
22677
22678    #[tokio::test]
22679    async fn workflow_start_options_send_server_enforced_deadlines() {
22680        let server = MockWorkerServer::start();
22681        let client = Client::builder(server.base_url())
22682            .timeout(Duration::from_secs(2))
22683            .build()
22684            .expect("client");
22685
22686        let handle = client
22687            .start_workflow_with_options(
22688                "rust.timeout",
22689                "rust-timeouts",
22690                "wf-start-options",
22691                WorkflowStartOptions::new()
22692                    .execution_timeout_seconds(30)
22693                    .run_timeout_seconds(1),
22694                json!([]),
22695            )
22696            .await
22697            .expect("workflow start");
22698
22699        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22700        let body = server.request_body("/api/workflows");
22701        assert_eq!(body["execution_timeout_seconds"], 30);
22702        assert_eq!(body["run_timeout_seconds"], 1);
22703
22704        let invalid = client
22705            .start_workflow_with_options(
22706                "rust.timeout",
22707                "rust-timeouts",
22708                "wf-invalid-options",
22709                WorkflowStartOptions::new()
22710                    .execution_timeout_seconds(1)
22711                    .run_timeout_seconds(2),
22712                json!([]),
22713            )
22714            .await
22715            .expect_err("invalid deadline ordering");
22716        assert!(invalid
22717            .to_string()
22718            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22719    }
22720
22721    #[tokio::test]
22722    async fn workflow_result_returns_each_typed_terminal_outcome() {
22723        let server = MockWorkerServer::start();
22724        let client = Client::builder(server.base_url())
22725            .timeout(Duration::from_secs(2))
22726            .build()
22727            .expect("client");
22728        let options = WorkflowResultOptions {
22729            poll_interval: Duration::ZERO,
22730            timeout: Duration::from_secs(1),
22731        };
22732
22733        let failed = WorkflowHandle {
22734            client: client.clone(),
22735            workflow_id: "wf-failed".to_string(),
22736            run_id: Some("run-failed".to_string()),
22737            workflow_type: "failure".to_string(),
22738        }
22739        .result(options)
22740        .await
22741        .expect_err("failed outcome");
22742        let Error::WorkflowFailed(failure) = failed else {
22743            panic!("expected WorkflowFailed");
22744        };
22745        assert_eq!(failure.workflow_id, "wf-failed");
22746        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22747        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22748        assert_eq!(failure.failure_category.as_deref(), Some("application"));
22749        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22750        assert_eq!(
22751            failure.exception_class.as_deref(),
22752            Some("billing::PaymentError")
22753        );
22754        assert_eq!(failure.non_retryable, Some(true));
22755
22756        for (workflow_id, expected_kind, expected_reason) in [
22757            (
22758                "wf-cancelled",
22759                WorkflowTerminalKind::Cancelled,
22760                "cleanup requested",
22761            ),
22762            (
22763                "wf-terminated",
22764                WorkflowTerminalKind::Terminated,
22765                "forced stop",
22766            ),
22767            (
22768                "wf-timed-out",
22769                WorkflowTerminalKind::TimedOut,
22770                "run_timeout",
22771            ),
22772        ] {
22773            let error = WorkflowHandle {
22774                client: client.clone(),
22775                workflow_id: workflow_id.to_string(),
22776                run_id: None,
22777                workflow_type: "terminal".to_string(),
22778            }
22779            .result(options)
22780            .await
22781            .expect_err("typed terminal outcome");
22782            let outcome = match error {
22783                Error::WorkflowCancelled(outcome) => outcome,
22784                Error::WorkflowTerminated(outcome) => outcome,
22785                Error::WorkflowTimedOut(outcome) => outcome,
22786                other => panic!("unexpected terminal error: {other}"),
22787            };
22788            assert_eq!(outcome.kind, expected_kind);
22789            assert_eq!(outcome.workflow_id, workflow_id);
22790            assert_eq!(outcome.reason, expected_reason);
22791        }
22792
22793        let wait_timeout = WorkflowHandle {
22794            client,
22795            workflow_id: "wf-waiting".to_string(),
22796            run_id: Some("run-waiting".to_string()),
22797            workflow_type: "waiting".to_string(),
22798        }
22799        .result(WorkflowResultOptions {
22800            poll_interval: Duration::ZERO,
22801            timeout: Duration::ZERO,
22802        })
22803        .await
22804        .expect_err("client wait timeout");
22805        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22806            panic!("expected typed client timeout");
22807        };
22808        assert_eq!(timeout.reason, "result_wait_timeout");
22809        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22810        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22811    }
22812
22813    #[tokio::test]
22814    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22815        let server = MockWorkerServer::start();
22816        let client = Client::builder(server.base_url())
22817            .timeout(Duration::from_secs(2))
22818            .build()
22819            .expect("client");
22820
22821        let handle = WorkflowHandle {
22822            client,
22823            workflow_id: "wf-selected".to_string(),
22824            run_id: Some("run-selected".to_string()),
22825            workflow_type: "selected".to_string(),
22826        };
22827        let options = WorkflowResultOptions {
22828            poll_interval: Duration::ZERO,
22829            timeout: Duration::from_secs(1),
22830        };
22831
22832        let current = handle
22833            .result(options)
22834            .await
22835            .expect("instance result follows the current run");
22836        assert_eq!(current, json!("current run output"));
22837
22838        let error = handle
22839            .result_selected_run(options)
22840            .await
22841            .expect_err("the selected run is cancelled even though the current run completed");
22842
22843        let Error::WorkflowCancelled(outcome) = error else {
22844            panic!("expected selected run cancellation");
22845        };
22846        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22847        assert_eq!(outcome.reason, "selected run cancelled");
22848        assert_eq!(
22849            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22850            1
22851        );
22852        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22853    }
22854
22855    #[tokio::test]
22856    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22857        let server = MockWorkerServer::draining_polls();
22858        let client = Client::builder(server.base_url())
22859            .timeout(Duration::from_secs(2))
22860            .build()
22861            .expect("client");
22862
22863        let workflow = client
22864            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22865            .await
22866            .expect("workflow drain response");
22867        let activity = client
22868            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22869            .await
22870            .expect("activity drain response");
22871        let query = client
22872            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22873            .await
22874            .expect("query drain response");
22875
22876        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22877            assert_eq!(
22878                outcome,
22879                WorkerPollOutcome::Stop {
22880                    poll_status: Some("draining".to_string()),
22881                    reason: Some("worker_draining".to_string()),
22882                }
22883            );
22884        }
22885
22886        assert!(client
22887            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22888            .await
22889            .expect("compatibility poll")
22890            .is_none());
22891    }
22892
22893    #[tokio::test]
22894    async fn managed_worker_honors_drain_stop_for_every_task_family() {
22895        let server = MockWorkerServer::draining_polls();
22896        let client = Client::builder(server.base_url())
22897            .timeout(Duration::from_secs(2))
22898            .build()
22899            .expect("client");
22900
22901        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22902            .worker_id("draining-workflow-worker")
22903            .poll_timeout(Duration::ZERO);
22904        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22905        workflow_worker
22906            .run()
22907            .await
22908            .expect("workflow drain is a clean stop");
22909
22910        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22911            .worker_id("draining-activity-worker")
22912            .poll_timeout(Duration::ZERO);
22913        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22914        activity_worker
22915            .run()
22916            .await
22917            .expect("activity drain is a clean stop");
22918
22919        let mut query_worker = Worker::new(client, "rust-workers")
22920            .worker_id("draining-query-worker")
22921            .poll_timeout(Duration::ZERO);
22922        query_worker.register_query("counter", "current", |_ctx, _args| async {
22923            Ok(Value::Null)
22924        });
22925        query_worker
22926            .run()
22927            .await
22928            .expect("query drain is a clean stop");
22929    }
22930
22931    #[tokio::test]
22932    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22933        let server = MockWorkerServer::start();
22934        let client = Client::builder(server.base_url())
22935            .timeout(Duration::from_secs(2))
22936            .build()
22937            .expect("client");
22938
22939        let heartbeat = client
22940            .heartbeat_activity_task(
22941                "activity-cancel",
22942                "attempt-cancel",
22943                "rust-worker",
22944                typed_fidelity_probe(),
22945            )
22946            .await
22947            .expect("cancellation heartbeat");
22948        assert!(heartbeat.cancel_requested);
22949        assert!(heartbeat.should_stop());
22950        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22951        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22952        let heartbeat_body =
22953            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22954        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22955        assert_eq!(
22956            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22957                .expect("typed heartbeat details"),
22958            typed_fidelity_probe()
22959        );
22960
22961        let error = client
22962            .complete_activity_task(
22963                "activity-cancel",
22964                "attempt-cancel",
22965                "rust-worker",
22966                json!({"late":true}),
22967                DEFAULT_CODEC,
22968            )
22969            .await
22970            .expect_err("late completion must be refused");
22971        assert!(activity_task_rejection_is_final(&error));
22972        let Error::ActivityTaskRejected(rejection) = error else {
22973            panic!("expected typed activity rejection");
22974        };
22975        assert_eq!(rejection.status, 409);
22976        assert_eq!(rejection.reason, "run_cancelled");
22977        assert!(rejection.cancel_requested);
22978        assert_eq!(rejection.can_continue, Some(false));
22979    }
22980
22981    #[tokio::test]
22982    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22983        let server = MockWorkerServer::cancelled_activity();
22984        let client = Client::builder(server.base_url())
22985            .timeout(Duration::from_secs(2))
22986            .build()
22987            .expect("client");
22988        let cancellation_observed = Arc::new(AtomicBool::new(false));
22989        let observed = Arc::clone(&cancellation_observed);
22990        let mut worker = Worker::new(client.clone(), "rust-workers")
22991            .worker_id("rust-cancel-worker")
22992            .poll_timeout(Duration::from_millis(10));
22993        worker.register_activity("cancel-aware", move |ctx, _args| {
22994            let observed = Arc::clone(&observed);
22995            async move {
22996                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22997                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22998                Ok(json!({"late":"completion"}))
22999            }
23000        });
23001
23002        assert_eq!(
23003            worker.run_once().await.expect("cancelled attempt handled"),
23004            1
23005        );
23006        assert!(cancellation_observed.load(Ordering::SeqCst));
23007        assert_eq!(
23008            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
23009            1
23010        );
23011
23012        let mut restarted = Worker::new(client, "rust-workers")
23013            .worker_id("rust-cancel-worker-restarted")
23014            .poll_timeout(Duration::from_millis(10));
23015        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
23016        assert_eq!(
23017            restarted
23018                .run_once()
23019                .await
23020                .expect("replacement worker continues polling"),
23021            0
23022        );
23023    }
23024
23025    #[tokio::test]
23026    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
23027        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
23028        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
23029        let client = Client::builder(server.base_url())
23030            .timeout(Duration::from_secs(2))
23031            .build()
23032            .expect("client");
23033
23034        let direct_error = client
23035            .complete_workflow_task(
23036                "workflow-timeout-task",
23037                "timeout-worker",
23038                3,
23039                vec![json!({
23040                    "type": "complete_workflow",
23041                    "result": fixture_envelope(Value::Null)
23042                })],
23043            )
23044            .await
23045            .expect_err("the low-level client preserves the completion rejection");
23046        let Error::Http { status, body } = direct_error else {
23047            panic!("expected the original HTTP completion rejection");
23048        };
23049        assert_eq!(status, reqwest::StatusCode::CONFLICT);
23050        assert_eq!(
23051            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
23052            "run_timed_out"
23053        );
23054
23055        let mut worker = Worker::new(client, "rust-workers")
23056            .worker_id("timeout-worker")
23057            .poll_timeout(Duration::from_millis(10));
23058        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
23059            Ok(json!({"late": "result"}))
23060        });
23061
23062        assert_eq!(
23063            worker
23064                .run_once()
23065                .await
23066                .expect("authoritative selected-run timeout settles the tick"),
23067            1
23068        );
23069        assert_eq!(
23070            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
23071            2,
23072            "both the direct client proof and managed worker must see the rejection"
23073        );
23074    }
23075
23076    #[tokio::test]
23077    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
23078        for (name, status, response) in [
23079            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
23080            (
23081                "command was recorded",
23082                "409 Conflict",
23083                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23084            ),
23085            (
23086                "lease conflict",
23087                "409 Conflict",
23088                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
23089            ),
23090            (
23091                "nonterminal run",
23092                "409 Conflict",
23093                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
23094            ),
23095            (
23096                "different selected run",
23097                "409 Conflict",
23098                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
23099            ),
23100            (
23101                "different task attempt",
23102                "409 Conflict",
23103                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23104            ),
23105            (
23106                "authentication failure",
23107                "401 Unauthorized",
23108                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23109            ),
23110            (
23111                "authorization failure",
23112                "403 Forbidden",
23113                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23114            ),
23115            (
23116                "protocol failure",
23117                "400 Bad Request",
23118                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
23119            ),
23120            (
23121                "malformed command",
23122                "422 Unprocessable Entity",
23123                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23124            ),
23125            (
23126                "transient server failure",
23127                "503 Service Unavailable",
23128                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23129            ),
23130        ] {
23131            let server = MockWorkerServer::workflow_completion(status, response);
23132            let client = Client::builder(server.base_url())
23133                .timeout(Duration::from_secs(2))
23134                .build()
23135                .expect("client");
23136            let mut worker = Worker::new(client, "rust-workers")
23137                .worker_id("timeout-worker")
23138                .poll_timeout(Duration::from_millis(10));
23139            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
23140                Ok(json!({"late": "result"}))
23141            });
23142
23143            let error = worker
23144                .run_once()
23145                .await
23146                .expect_err(&format!("{name} must remain an error"));
23147            assert!(
23148                matches!(error, Error::Http { .. } | Error::Protocol(_)),
23149                "{name} returned an unexpected error variant: {error}"
23150            );
23151        }
23152    }
23153
23154    #[tokio::test]
23155    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
23156        let server = MockWorkerServer::start();
23157        let client = Client::builder(server.base_url())
23158            .worker_token(Some("worker-secret".to_string()))
23159            .namespace("orders")
23160            .timeout(Duration::from_secs(2))
23161            .build()
23162            .expect("client");
23163        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
23164
23165        let result = client
23166            .deregister_worker_registration("worker/α space")
23167            .await
23168            .expect("deregister worker registration");
23169
23170        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
23171        assert_eq!(
23172            server.worker_protocol_for(path).as_deref(),
23173            Some(WORKER_PROTOCOL_VERSION)
23174        );
23175        assert_eq!(server.control_protocol_for(path), None);
23176        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
23177        assert_eq!(
23178            server.authorization_for(path).as_deref(),
23179            Some("Bearer worker-secret")
23180        );
23181        assert_eq!(
23182            result,
23183            WorkerDeregistrationEnvelope {
23184                worker_id: "deregistered-worker".to_string(),
23185                outcome: "deregistered".to_string(),
23186                recovered_workflow_task_count: 2,
23187            }
23188        );
23189    }
23190
23191    #[tokio::test]
23192    async fn low_level_registration_rejects_update_validators_before_transport() {
23193        let server = MockWorkerServer::start();
23194        let client = Client::builder(server.base_url())
23195            .timeout(Duration::from_secs(2))
23196            .build()
23197            .expect("client");
23198
23199        for update_validators in [json!(["approve"]), json!("approve")] {
23200            let error = client
23201                .register_worker_with_command_contracts(
23202                    "validator-claiming-worker",
23203                    "rust-workers",
23204                    vec!["orders".to_string()],
23205                    vec![],
23206                    1,
23207                    1,
23208                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
23209                    json!({
23210                        "orders": {
23211                            "queries": ["current"],
23212                            "updates": ["approve"],
23213                            "update_validators": update_validators,
23214                        },
23215                    }),
23216                )
23217                .await
23218                .expect_err("unsupported validator claims must fail before registration");
23219
23220            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
23221                panic!("expected typed unsupported-validator failure");
23222            };
23223            assert_eq!(workflow_type, "orders");
23224        }
23225        assert_eq!(server.request_count("/api/worker/register"), 0);
23226    }
23227
23228    #[tokio::test]
23229    async fn low_level_registration_preserves_query_and_update_contracts() {
23230        let server = MockWorkerServer::start();
23231        let client = Client::builder(server.base_url())
23232            .timeout(Duration::from_secs(2))
23233            .build()
23234            .expect("client");
23235        let contracts = json!({
23236            "orders": {
23237                "queries": ["current"],
23238                "updates": ["approve"],
23239                "update_validators": [],
23240            },
23241            "payments": {
23242                "queries": ["status"],
23243                "updates": ["capture"],
23244            },
23245        });
23246
23247        client
23248            .register_worker_with_command_contracts(
23249                "command-worker",
23250                "rust-workers",
23251                vec!["orders".to_string(), "payments".to_string()],
23252                vec![],
23253                1,
23254                1,
23255                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
23256                contracts.clone(),
23257            )
23258            .await
23259            .expect("query and update contracts must remain supported");
23260
23261        assert_eq!(
23262            server.request_body("/api/worker/register")["workflow_command_contracts"],
23263            contracts
23264        );
23265        assert!(server
23266            .request_body("/api/worker/register")
23267            .get("workflow_definition_fingerprints")
23268            .is_none());
23269    }
23270
23271    #[tokio::test]
23272    async fn rust_worker_advertises_source_backed_definition_and_clears_it_on_reregistration() {
23273        let server = MockWorkerServer::start();
23274        let client = Client::builder(server.base_url())
23275            .timeout(Duration::from_secs(2))
23276            .build()
23277            .expect("client");
23278        let mut worker = Worker::new(client, "rust-workers");
23279        worker.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
23280
23281        assert!(worker
23282            .set_workflow_definition_sources("missing", &[include_str!("lib.rs")])
23283            .is_err());
23284        assert!(worker
23285            .set_workflow_definition_sources("orders", &[])
23286            .is_err());
23287        worker
23288            .set_workflow_definition_sources("orders", &[include_str!("lib.rs")])
23289            .expect("embed workflow source");
23290        worker.register().await.expect("register worker");
23291        let first = server.request_bodies("/api/worker/register")[0]
23292            ["workflow_definition_fingerprints"]["orders"]
23293            .as_str()
23294            .expect("source-backed fingerprint")
23295            .to_string();
23296        assert!(first.starts_with("sha256:"));
23297
23298        worker
23299            .set_workflow_definition_sources("orders", &[include_str!("lib.rs"), "changed helper"])
23300            .expect("change embedded helper source");
23301        worker.register().await.expect("register changed source");
23302        let changed = server.request_bodies("/api/worker/register")[1]
23303            ["workflow_definition_fingerprints"]["orders"]
23304            .as_str()
23305            .expect("changed fingerprint")
23306            .to_string();
23307        assert_ne!(first, changed);
23308
23309        worker.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
23310        worker
23311            .register()
23312            .await
23313            .expect("register replacement handler");
23314        assert_eq!(
23315            server.request_bodies("/api/worker/register")[2]["workflow_definition_fingerprints"],
23316            json!({})
23317        );
23318    }
23319
23320    #[tokio::test]
23321    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
23322        let server = MockWorkerServer::start();
23323        let control_only = Client::builder(server.base_url())
23324            .control_token(Some("control-secret".to_string()))
23325            .build()
23326            .expect("control client");
23327
23328        let error = control_only
23329            .register_worker("worker", "queue", vec![], vec![], 1, 1)
23330            .await
23331            .expect_err("control token must not authorize a worker request");
23332        assert!(matches!(
23333            error,
23334            Error::MissingRoleCredentials { role: "worker", .. }
23335        ));
23336        assert_eq!(server.request_count("/api/worker/register"), 0);
23337
23338        let worker_only = Client::builder(server.base_url())
23339            .worker_token(Some("worker-secret".to_string()))
23340            .build()
23341            .expect("worker client");
23342        let error = worker_only
23343            .health()
23344            .await
23345            .expect_err("worker token must not authorize a control request");
23346        assert!(matches!(
23347            error,
23348            Error::MissingRoleCredentials {
23349                role: "control",
23350                ..
23351            }
23352        ));
23353        assert_eq!(server.request_count("/api/health"), 0);
23354    }
23355
23356    #[tokio::test]
23357    async fn shared_token_supports_worker_and_control_planes() {
23358        let server = MockWorkerServer::start();
23359        let client = Client::builder(server.base_url())
23360            .token(Some("shared-secret".to_string()))
23361            .build()
23362            .expect("client");
23363
23364        client.health().await.expect("control request");
23365        client
23366            .register_worker("worker", "queue", vec![], vec![], 1, 1)
23367            .await
23368            .expect("worker request");
23369
23370        assert_eq!(
23371            server.authorization_for("/api/health").as_deref(),
23372            Some("Bearer shared-secret")
23373        );
23374        assert_eq!(
23375            server.control_protocol_for("/api/health").as_deref(),
23376            Some(CONTROL_PLANE_VERSION)
23377        );
23378        assert_eq!(
23379            server.authorization_for("/api/worker/register").as_deref(),
23380            Some("Bearer shared-secret")
23381        );
23382        assert_eq!(
23383            server
23384                .worker_protocol_for("/api/worker/register")
23385                .as_deref(),
23386            Some(WORKER_PROTOCOL_VERSION)
23387        );
23388    }
23389
23390    #[tokio::test]
23391    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
23392        let server = MockWorkerServer::start();
23393        let client = Client::builder(server.base_url())
23394            .timeout(Duration::from_secs(2))
23395            .build()
23396            .expect("client");
23397
23398        client
23399            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
23400            .await
23401            .expect("register");
23402        client
23403            .heartbeat_worker("capture-worker", 1, 1)
23404            .await
23405            .expect("heartbeat");
23406        client
23407            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23408            .await
23409            .expect("workflow poll");
23410        client
23411            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23412            .await
23413            .expect("activity poll");
23414
23415        for path in [
23416            "/api/worker/register",
23417            "/api/worker/heartbeat",
23418            "/api/worker/workflow-tasks/poll",
23419            "/api/worker/activity-tasks/poll",
23420        ] {
23421            assert_eq!(
23422                server.worker_protocol_for(path).as_deref(),
23423                Some(WORKER_PROTOCOL_VERSION),
23424                "unexpected protocol for {path}"
23425            );
23426        }
23427
23428        assert_eq!(
23429            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
23430            1
23431        );
23432        assert_eq!(
23433            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
23434            1
23435        );
23436        assert!(
23437            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
23438                .as_str()
23439                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
23440        );
23441        assert!(
23442            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
23443                .as_str()
23444                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
23445        );
23446    }
23447
23448    #[tokio::test]
23449    async fn query_task_endpoints_send_the_query_feature_protocol() {
23450        let server = MockWorkerServer::start();
23451        let client = Client::builder(server.base_url())
23452            .timeout(Duration::from_secs(2))
23453            .build()
23454            .expect("client");
23455
23456        client
23457            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23458            .await
23459            .expect("query poll");
23460        client
23461            .complete_query_task(
23462                "query-capture",
23463                "capture-worker",
23464                1,
23465                json!(8),
23466                DEFAULT_CODEC,
23467            )
23468            .await
23469            .expect("query complete");
23470        client
23471            .fail_query_task(
23472                "query-capture",
23473                "capture-worker",
23474                1,
23475                "failed",
23476                "query_rejected",
23477                "QueryFailed",
23478            )
23479            .await
23480            .expect("query fail");
23481
23482        for path in [
23483            "/api/worker/query-tasks/poll",
23484            "/api/worker/query-tasks/query-capture/complete",
23485            "/api/worker/query-tasks/query-capture/fail",
23486        ] {
23487            assert_eq!(
23488                server.worker_protocol_for(path).as_deref(),
23489                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
23490                "unexpected protocol for {path}"
23491            );
23492        }
23493
23494        assert_eq!(
23495            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
23496            1
23497        );
23498        assert!(
23499            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
23500                .as_str()
23501                .is_some_and(|id| id.starts_with("rust-query-poll-"))
23502        );
23503    }
23504
23505    #[tokio::test]
23506    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
23507        let server = MockWorkerServer::transient_worker_failures();
23508        let client = Client::builder(server.base_url())
23509            .timeout(Duration::from_secs(2))
23510            .build()
23511            .expect("client");
23512
23513        client
23514            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23515            .await
23516            .expect("workflow poll retry");
23517        client
23518            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23519            .await
23520            .expect("activity poll retry");
23521        client
23522            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23523            .await
23524            .expect("query poll retry");
23525
23526        for path in [
23527            "/api/worker/workflow-tasks/poll",
23528            "/api/worker/activity-tasks/poll",
23529            "/api/worker/query-tasks/poll",
23530        ] {
23531            let bodies = server.request_bodies(path);
23532            assert_eq!(bodies.len(), 2, "{path} must be retried once");
23533            assert_eq!(
23534                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
23535                "{path} must preserve the request binding across retry"
23536            );
23537        }
23538    }
23539
23540    #[tokio::test]
23541    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
23542        let server = MockWorkerServer::consecutive_poll_failures(2);
23543        let client = Client::builder(server.base_url())
23544            .timeout(Duration::from_secs(2))
23545            .build()
23546            .expect("client");
23547        let mut worker = Worker::new(client, "capture")
23548            .worker_id("capture-worker")
23549            .poll_timeout(Duration::from_millis(10))
23550            .retry_policy(WorkerRetryPolicy {
23551                max_retries: 2,
23552                initial_backoff: Duration::from_millis(1),
23553                max_backoff: Duration::from_millis(1),
23554            });
23555        worker.register_workflow(
23556            "capture.workflow",
23557            |_ctx, _input| async move { Ok(Value::Null) },
23558        );
23559        worker.register_activity(
23560            "capture.activity",
23561            |_ctx, _input| async move { Ok(Value::Null) },
23562        );
23563        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23564            Ok(Value::Null)
23565        });
23566
23567        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23568
23569        for path in [
23570            "/api/worker/workflow-tasks/poll",
23571            "/api/worker/activity-tasks/poll",
23572            "/api/worker/query-tasks/poll",
23573        ] {
23574            let bodies = server.request_bodies(path);
23575            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23576            assert!(
23577                bodies
23578                    .iter()
23579                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23580                "{path} must preserve one request binding across every retry"
23581            );
23582        }
23583    }
23584
23585    fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23586        let reason = if unavailable {
23587            "storage_admission_unavailable"
23588        } else {
23589            "storage_pressure"
23590        };
23591        let mut body = json!({
23592            "reason": reason,
23593            "storage_state": if unavailable { "fenced" } else { "draining" },
23594            "retryable": true,
23595            "retry_after_seconds": 1,
23596        });
23597        if !mid_poll {
23598            body["request_admitted"] = json!(false);
23599        }
23600        if let Some(id) = poll_id {
23601            body["task"] = Value::Null;
23602            body["poll_status"] = json!(reason);
23603            body["poll_request_id"] = json!(id);
23604            body["retry_same_poll_request_id"] = json!(true);
23605            body["claim_admitted"] = json!(false);
23606        }
23607        body
23608    }
23609
23610    fn backend_refusal(path: &str, request: &str) -> Option<Value> {
23611        let operation = match path {
23612            "/api/worker/register" => "register_worker",
23613            "/api/worker/heartbeat" => "heartbeat_worker",
23614            "/api/worker/workflow-tasks/poll" => "poll_workflow_task",
23615            "/api/worker/activity-tasks/poll" => "poll_activity_task",
23616            "/api/worker/query-tasks/poll" => "poll_query_task",
23617            "/api/worker/update-validation-tasks/poll" => "poll_update_validation_task",
23618            _ => return None,
23619        };
23620        let request: Value = serde_json::from_str(request).ok()?;
23621        let mut response = json!({
23622            "reason": "backend_unavailable",
23623            "operation": operation,
23624            "outcome": "unknown",
23625            "retryable": true,
23626            "retry_after_seconds": 1,
23627            "worker_id": request["worker_id"],
23628            "task_queue": request.get("task_queue"),
23629        });
23630        if path.ends_with("/poll") {
23631            response["task"] = Value::Null;
23632            response["poll_status"] = json!("backend_unavailable");
23633            response["poll_request_id"] = request["poll_request_id"].clone();
23634            response["retry_same_poll_request_id"] = json!(true);
23635        }
23636        Some(response)
23637    }
23638
23639    fn backend_retry_override(
23640        path: &str,
23641        request: &str,
23642        number: usize,
23643    ) -> Option<(&'static str, String)> {
23644        if number > 8 {
23645            return None;
23646        }
23647        Some((
23648            "503 Service Unavailable",
23649            backend_refusal(path, request)?.to_string(),
23650        ))
23651    }
23652
23653    fn backend_unavailable_override(
23654        path: &str,
23655        request: &str,
23656        _number: usize,
23657    ) -> Option<(&'static str, String)> {
23658        Some((
23659            "503 Service Unavailable",
23660            backend_refusal(path, request)?.to_string(),
23661        ))
23662    }
23663
23664    fn storage_worker(server: &MockWorkerServer) -> Worker {
23665        Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23666            .worker_id("storage-worker")
23667            .retry_policy(WorkerRetryPolicy {
23668                max_retries: 1,
23669                initial_backoff: Duration::from_millis(1),
23670                max_backoff: Duration::from_millis(1),
23671            })
23672    }
23673
23674    fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23675        let requests = server.requests.lock().expect("requests");
23676        let bodies: Vec<_> = requests
23677            .iter()
23678            .filter(|request| request.path == path)
23679            .map(|request| &request.body)
23680            .collect();
23681        assert_eq!(bodies.len(), count, "{path}");
23682        assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23683    }
23684
23685    #[test]
23686    fn backend_recovery_requires_the_explicit_worker_contract() {
23687        for (path, operation) in [
23688            ("/worker/register", "register_worker"),
23689            ("/worker/heartbeat", "heartbeat_worker"),
23690            ("/worker/workflow-tasks/poll", "poll_workflow_task"),
23691            ("/worker/activity-tasks/poll", "poll_activity_task"),
23692            ("/worker/query-tasks/poll", "poll_query_task"),
23693            (
23694                "/worker/update-validation-tasks/poll",
23695                "poll_update_validation_task",
23696            ),
23697        ] {
23698            let poll_id = path.ends_with("/poll").then_some("same-poll");
23699            let request = json!({"worker_id":"same-worker","task_queue":"same-queue","poll_request_id":poll_id});
23700            let body = backend_refusal(&format!("/api{path}"), &request.to_string())
23701                .expect("worker operation");
23702            assert_eq!(body["operation"], operation);
23703            let error = Error::Http {
23704                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23705                body: body.to_string(),
23706            };
23707            assert_eq!(
23708                worker_backend_unavailable_retry_after(&error, path, poll_id),
23709                Some(Duration::from_secs(1))
23710            );
23711            assert!(!worker_operation_is_retryable(&error));
23712            for (field, value) in [
23713                ("operation", json!("wrong_operation")),
23714                ("outcome", json!("completed")),
23715                ("retryable", json!(false)),
23716                ("retry_after_seconds", json!(0)),
23717                ("worker_id", json!("")),
23718            ] {
23719                let mut invalid = body.clone();
23720                invalid[field] = value;
23721                let error = Error::Http {
23722                    status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23723                    body: invalid.to_string(),
23724                };
23725                assert!(
23726                    worker_backend_unavailable_retry_after(&error, path, poll_id).is_none(),
23727                    "{path}: {field}"
23728                );
23729                assert!(!worker_operation_is_retryable(&error));
23730            }
23731            if let Some(poll_id) = poll_id {
23732                for (field, value) in [
23733                    ("poll_request_id", json!("wrong-poll")),
23734                    ("poll_status", json!("empty")),
23735                    ("task", json!({"task_id":"claimed"})),
23736                    ("retry_same_poll_request_id", json!(false)),
23737                ] {
23738                    let mut invalid = body.clone();
23739                    invalid[field] = value;
23740                    let error = Error::Http {
23741                        status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23742                        body: invalid.to_string(),
23743                    };
23744                    assert!(
23745                        worker_backend_unavailable_retry_after(&error, path, Some(poll_id))
23746                            .is_none(),
23747                        "{path}: {field}"
23748                    );
23749                }
23750            }
23751            let error = Error::Http {
23752                status: reqwest::StatusCode::UNAUTHORIZED,
23753                body: body.to_string(),
23754            };
23755            assert!(worker_backend_unavailable_retry_after(&error, path, poll_id).is_none());
23756            assert!(!worker_operation_is_retryable(&error));
23757        }
23758    }
23759
23760    #[tokio::test]
23761    async fn backend_recovery_outlives_generic_retry_budget_and_preserves_polls() {
23762        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23763            request_override: Some(backend_retry_override),
23764            ..MockWorkerBehavior::default()
23765        });
23766        let mut worker =
23767            storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23768        worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23769        worker.register_activity("backend.activity", |_, _| async { Ok(Value::Null) });
23770        worker.register_query("backend.workflow", "state", |_, _| async {
23771            Ok(Value::Null)
23772        });
23773        worker.register().await.expect("registration recovery");
23774        worker
23775            .client
23776            .heartbeat_worker("storage-worker", 1, 1)
23777            .await
23778            .expect("heartbeat recovery");
23779        assert_eq!(worker.run_once().await.expect("poll recovery"), 0);
23780        for path in [
23781            "/api/worker/register",
23782            "/api/worker/heartbeat",
23783            "/api/worker/workflow-tasks/poll",
23784            "/api/worker/activity-tasks/poll",
23785            "/api/worker/query-tasks/poll",
23786        ] {
23787            assert_identical_requests(&server, path, 9);
23788        }
23789    }
23790
23791    #[tokio::test]
23792    async fn backend_recovery_wait_stops_with_worker() {
23793        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23794            request_override: Some(backend_unavailable_override),
23795            ..MockWorkerBehavior::default()
23796        });
23797        let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy {
23798            max_retries: 1,
23799            initial_backoff: Duration::from_secs(2),
23800            max_backoff: Duration::from_secs(2),
23801        });
23802        worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23803        let started = tokio::time::Instant::now();
23804        let _ = tokio::time::timeout(
23805            Duration::from_secs(1),
23806            worker.run_until(tokio::time::sleep(Duration::from_millis(100))),
23807        )
23808        .await
23809        .expect("shutdown interrupts backend wait");
23810        assert!(started.elapsed() < Duration::from_secs(1));
23811        assert!(server.request_count("/api/worker/register") >= 1);
23812    }
23813
23814    #[test]
23815    fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23816        for unavailable in [false, true] {
23817            for mid_poll in [false, true] {
23818                let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23819                let error = Error::Http {
23820                    status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23821                    body: body.to_string(),
23822                };
23823                assert_eq!(
23824                    worker_storage_admission_retry_after(&error, Some("same-poll")),
23825                    Some(Duration::from_secs(1))
23826                );
23827                assert!(
23828                    !worker_operation_is_retryable(&error),
23829                    "storage is not a bounded generic retry"
23830                );
23831                for (field, value) in [
23832                    ("poll_request_id", json!("wrong-poll")),
23833                    ("task", json!({"task_id":"claimed"})),
23834                    ("retryable", json!(false)),
23835                    ("retry_after_seconds", json!(0)),
23836                    ("retry_after_seconds", json!(true)),
23837                    ("retry_after_seconds", json!(1.0)),
23838                    ("storage_state", json!("normal")),
23839                    ("poll_status", json!("empty")),
23840                    ("claim_admitted", json!(true)),
23841                    ("retry_same_poll_request_id", json!(false)),
23842                    ("request_admitted", json!(true)),
23843                ] {
23844                    let mut invalid = body.clone();
23845                    invalid[field] = value;
23846                    let error = Error::Http {
23847                        status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23848                        body: invalid.to_string(),
23849                    };
23850                    assert!(
23851                        worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23852                        "{field}"
23853                    );
23854                }
23855            }
23856        }
23857        let body = storage_refusal(None, false, false);
23858        let error = Error::Http {
23859            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23860            body: body.to_string(),
23861        };
23862        assert!(worker_storage_admission_retry_after(&error, None).is_some());
23863        assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23864        let error = Error::Http {
23865            status: reqwest::StatusCode::FORBIDDEN,
23866            body: body.to_string(),
23867        };
23868        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23869        let body = storage_refusal(None, false, true);
23870        let error = Error::Http {
23871            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23872            body: body.to_string(),
23873        };
23874        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23875    }
23876
23877    #[tokio::test]
23878    async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23879        for unavailable in [false, true] {
23880            for mid_poll in [false, true] {
23881                let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23882                    poll_failures_per_path: 1,
23883                    storage_refusals: 7,
23884                    storage_path: Some("/poll"),
23885                    storage_unavailable: unavailable,
23886                    storage_mid_poll: mid_poll,
23887                    ..MockWorkerBehavior::default()
23888                });
23889                let mut worker = storage_worker(&server);
23890                worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23891                assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23892                for path in [
23893                    "/api/worker/workflow-tasks/poll",
23894                    "/api/worker/activity-tasks/poll",
23895                    "/api/worker/query-tasks/poll",
23896                ] {
23897                    assert_identical_requests(&server, path, 9);
23898                }
23899            }
23900        }
23901    }
23902
23903    #[tokio::test]
23904    async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23905        struct CountedBody(Arc<AtomicUsize>);
23906        impl Serialize for CountedBody {
23907            fn serialize<S: Serializer>(
23908                &self,
23909                serializer: S,
23910            ) -> std::result::Result<S::Ok, S::Error> {
23911                let count = self.0.fetch_add(1, Ordering::SeqCst);
23912                json!({"serialization":count,"lease_owner":"worker","attempt":7})
23913                    .serialize(serializer)
23914            }
23915        }
23916        for path in [
23917            "/api/worker/register",
23918            "/api/worker/heartbeat",
23919            "/api/worker/workflow-tasks/storage-task/complete",
23920            "/api/worker/workflow-tasks/storage-task/fail",
23921            "/api/worker/activity-tasks/storage-task/complete",
23922            "/api/worker/activity-tasks/storage-task/fail",
23923            "/api/worker/activity-tasks/storage-task/heartbeat",
23924            "/api/worker/query-tasks/storage-task/complete",
23925            "/api/worker/query-tasks/storage-task/fail",
23926        ] {
23927            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23928                storage_refusals: 7,
23929                storage_path: Some(path),
23930                ..MockWorkerBehavior::default()
23931            });
23932            let worker =
23933                storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23934            let calls = Arc::new(AtomicUsize::new(0));
23935            let _: Value = worker
23936                .client
23937                .request_json(
23938                    reqwest::Method::POST,
23939                    &path[4..],
23940                    RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23941                    Some(&CountedBody(Arc::clone(&calls))),
23942                )
23943                .await
23944                .expect("prepared request recovery");
23945            assert_eq!(calls.load(Ordering::SeqCst), 1);
23946            assert_identical_requests(&server, path, 8);
23947        }
23948        for worker_scope in [false, true] {
23949            let path = if worker_scope {
23950                "/api/health"
23951            } else {
23952                "/api/worker/register"
23953            };
23954            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23955                storage_refusals: usize::MAX,
23956                storage_path: Some(path),
23957                ..MockWorkerBehavior::default()
23958            });
23959            let worker = storage_worker(&server);
23960            let client = worker.client.clone();
23961            let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
23962            let error = if worker_scope {
23963                worker
23964                    .client
23965                    .health()
23966                    .await
23967                    .expect_err("control plane is not retried")
23968            } else {
23969                client
23970                    .request_json::<Value, Value>(
23971                        reqwest::Method::POST,
23972                        "/worker/register",
23973                        RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23974                        Some(&json!({})),
23975                    )
23976                    .await
23977                    .expect_err("direct client is not retried")
23978            };
23979            assert!(worker_storage_admission_body(&error).is_some());
23980            assert_eq!(server.request_count(path), 1);
23981        }
23982    }
23983
23984    #[tokio::test]
23985    async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
23986        for fail in [false, true] {
23987            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23988                storage_activity: true,
23989                storage_refusals: 7,
23990                storage_path: Some("/storage-activity/"),
23991                ..MockWorkerBehavior::default()
23992            });
23993            let mut worker = storage_worker(&server);
23994            let calls = Arc::new(AtomicUsize::new(0));
23995            let observed = Arc::clone(&calls);
23996            worker.register_activity("storage.activity", move |ctx, _| {
23997                let calls = Arc::clone(&observed);
23998                async move {
23999                    calls.fetch_add(1, Ordering::SeqCst);
24000                    ctx.heartbeat(json!({"step":2})).await?;
24001                    if fail {
24002                        Err(Error::WorkerLoop("intentional handler failure".to_string()))
24003                    } else {
24004                        Ok(json!({"receipt":true}))
24005                    }
24006                }
24007            });
24008            assert_eq!(worker.run_once().await.expect("activity settled"), 1);
24009            assert_eq!(calls.load(Ordering::SeqCst), 1);
24010            assert_identical_requests(
24011                &server,
24012                "/api/worker/activity-tasks/storage-activity/heartbeat",
24013                8,
24014            );
24015            let suffix = if fail { "fail" } else { "complete" };
24016            assert_identical_requests(
24017                &server,
24018                &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
24019                8,
24020            );
24021            let other = if fail { "complete" } else { "fail" };
24022            assert_eq!(
24023                server.request_count(&format!(
24024                    "/api/worker/activity-tasks/storage-activity/{other}"
24025                )),
24026                0
24027            );
24028        }
24029    }
24030
24031    #[tokio::test]
24032    async fn storage_waits_are_interruptible_without_false_activity_failure() {
24033        for path in [
24034            "/api/worker/register",
24035            "/api/worker/heartbeat",
24036            "/api/worker/activity-tasks/poll",
24037            "/api/worker/activity-tasks/storage-activity/heartbeat",
24038            "/api/worker/activity-tasks/storage-activity/complete",
24039        ] {
24040            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24041                storage_activity: true,
24042                storage_refusals: usize::MAX,
24043                storage_path: Some(path),
24044                ..MockWorkerBehavior::default()
24045            });
24046            let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
24047            worker.register_activity("storage.activity", |ctx, _| async move {
24048                ctx.heartbeat(json!({"step":2})).await?;
24049                Ok(json!({"receipt":true}))
24050            });
24051            let shutdown = async {
24052                while server.request_count(path) == 0 {
24053                    tokio::time::sleep(Duration::from_millis(1)).await;
24054                }
24055            };
24056            let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
24057                .await
24058                .expect("shutdown interrupts admission");
24059            assert!(
24060                result.is_err(),
24061                "a refused operation must not appear acknowledged: {path}, {result:?}"
24062            );
24063            assert_eq!(server.request_count(path), 1);
24064            assert_eq!(
24065                server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
24066                0
24067            );
24068            assert_eq!(
24069                server.request_count("/api/worker/registrations/mock-worker"),
24070                usize::from(!path.ends_with("/register"))
24071            );
24072        }
24073    }
24074
24075    #[tokio::test]
24076    async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
24077        for fail in [false, true] {
24078            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24079                storage_query: true,
24080                storage_refusals: 7,
24081                storage_path: Some("/storage-query/"),
24082                ..MockWorkerBehavior::default()
24083            });
24084            let mut worker = storage_worker(&server);
24085            worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
24086            let calls = Arc::new(AtomicUsize::new(0));
24087            let observed = Arc::clone(&calls);
24088            worker.register_query("storage.workflow", "state", move |_, _| {
24089                let calls = Arc::clone(&observed);
24090                async move {
24091                    calls.fetch_add(1, Ordering::SeqCst);
24092                    if fail {
24093                        Err(Error::WorkerLoop("intentional query failure".to_string()))
24094                    } else {
24095                        Ok(json!({"state":"waiting"}))
24096                    }
24097                }
24098            });
24099            assert_eq!(worker.run_once().await.expect("query settled"), 1);
24100            assert_eq!(calls.load(Ordering::SeqCst), 1);
24101            let suffix = if fail { "fail" } else { "complete" };
24102            assert_identical_requests(
24103                &server,
24104                &format!("/api/worker/query-tasks/storage-query/{suffix}"),
24105                8,
24106            );
24107            let other = if fail { "complete" } else { "fail" };
24108            assert_eq!(
24109                server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
24110                0
24111            );
24112        }
24113    }
24114
24115    #[tokio::test]
24116    async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
24117        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24118            storage_refusals: 7,
24119            storage_path: Some("/poll"),
24120            unauthorized_polls: true,
24121            ..MockWorkerBehavior::default()
24122        });
24123        let error = storage_worker(&server)
24124            .run_once()
24125            .await
24126            .expect_err("auth remains terminal");
24127        assert!(matches!(
24128            error,
24129            Error::Http {
24130                status: reqwest::StatusCode::UNAUTHORIZED,
24131                ..
24132            }
24133        ));
24134        assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
24135
24136        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24137            storage_refusals: 7,
24138            storage_path: Some("/activity-cancel/complete"),
24139            ..MockWorkerBehavior::default()
24140        });
24141        let worker =
24142            storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
24143        let error = worker
24144            .client
24145            .complete_activity_task(
24146                "activity-cancel",
24147                "attempt-cancel",
24148                "worker",
24149                json!({}),
24150                DEFAULT_CODEC,
24151            )
24152            .await
24153            .expect_err("cancellation remains terminal");
24154        assert!(activity_task_rejection_is_final(&error));
24155        assert_identical_requests(
24156            &server,
24157            "/api/worker/activity-tasks/activity-cancel/complete",
24158            8,
24159        );
24160
24161        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24162            storage_refusals: usize::MAX,
24163            storage_path: Some("/poll"),
24164            storage_wrong_poll_id: true,
24165            ..MockWorkerBehavior::default()
24166        });
24167        assert!(storage_worker(&server).run_once().await.is_err());
24168        assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
24169    }
24170
24171    #[tokio::test]
24172    async fn storage_pollers_stop_when_the_run_future_is_aborted() {
24173        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24174            storage_refusals: usize::MAX,
24175            storage_path: Some("/poll"),
24176            ..MockWorkerBehavior::default()
24177        });
24178        let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
24179        worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
24180        let run = tokio::spawn(async move { worker.run().await });
24181        tokio::time::timeout(Duration::from_secs(2), async {
24182            while server.request_count("/api/worker/activity-tasks/poll") == 0 {
24183                tokio::time::sleep(Duration::from_millis(1)).await;
24184            }
24185        })
24186        .await
24187        .expect("poll started");
24188        run.abort();
24189        assert!(run.await.expect_err("cancelled run").is_cancelled());
24190        tokio::time::sleep(Duration::from_millis(250)).await;
24191        assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
24192    }
24193
24194    #[tokio::test]
24195    async fn query_protocol_rejection_from_older_server_is_typed() {
24196        let server = MockWorkerServer::reject_query_protocol();
24197        let client = Client::builder(server.base_url())
24198            .timeout(Duration::from_secs(2))
24199            .build()
24200            .expect("client");
24201
24202        let error = client
24203            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
24204            .await
24205            .expect_err("server below query protocol floor must reject");
24206        let Error::Protocol(failure) = error else {
24207            panic!("expected typed protocol failure");
24208        };
24209
24210        assert_eq!(failure.status, 400);
24211        assert_eq!(failure.reason, "unsupported_protocol_version");
24212        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
24213        assert_eq!(
24214            failure.requested_version.as_deref(),
24215            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
24216        );
24217        assert_eq!(
24218            server
24219                .worker_protocol_for("/api/worker/query-tasks/poll")
24220                .as_deref(),
24221            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
24222        );
24223    }
24224
24225    #[tokio::test]
24226    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
24227        let server = MockWorkerServer::reject_query_protocol();
24228        let client = Client::builder(server.base_url())
24229            .timeout(Duration::from_secs(2))
24230            .build()
24231            .expect("client");
24232        let mut worker = Worker::new(client, "rust-workers")
24233            .worker_id("baseline-worker")
24234            .poll_timeout(Duration::from_millis(10));
24235
24236        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
24237            Ok(Value::Null)
24238        });
24239
24240        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
24241        assert_eq!(
24242            server
24243                .worker_protocol_for("/api/worker/workflow-tasks/poll")
24244                .as_deref(),
24245            Some(WORKER_PROTOCOL_VERSION)
24246        );
24247        assert_eq!(
24248            server.worker_protocol_for("/api/worker/query-tasks/poll"),
24249            None,
24250            "a worker without query handlers must not use the query-task endpoint"
24251        );
24252    }
24253
24254    #[tokio::test]
24255    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
24256        let server = MockWorkerServer::reject_query_completion();
24257        let client = Client::builder(server.base_url())
24258            .timeout(Duration::from_secs(2))
24259            .build()
24260            .expect("client");
24261
24262        let error = client
24263            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
24264            .await
24265            .expect_err("expired completion must be rejected");
24266        let Error::QueryFailed(failure) = error else {
24267            panic!("expected typed query failure");
24268        };
24269        assert_eq!(failure.status, 409);
24270        assert_eq!(failure.reason, "query_task_timed_out");
24271
24272        let mut worker = Worker::new(client, "rust-workers")
24273            .worker_id("late-worker")
24274            .poll_timeout(Duration::from_millis(10));
24275        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24276        worker.register_query(
24277            "counter",
24278            "current",
24279            |_ctx, _args| async move { Ok(json!(8)) },
24280        );
24281
24282        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
24283        assert_eq!(
24284            worker
24285                .run_once()
24286                .await
24287                .expect("worker continues after late completion"),
24288            0
24289        );
24290        assert_eq!(
24291            server.request_count("/api/worker/query-tasks/query-late/complete"),
24292            2
24293        );
24294        assert_eq!(
24295            server.request_count("/api/worker/query-tasks/query-late/fail"),
24296            0,
24297            "a server completion rejection must not be reported as an encoding failure"
24298        );
24299    }
24300
24301    #[tokio::test]
24302    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
24303        let server = MockWorkerServer::start();
24304        let client = Client::builder(server.base_url())
24305            .timeout(Duration::from_secs(2))
24306            .build()
24307            .expect("client");
24308        let mut worker = Worker::new(client, "rust-workers")
24309            .worker_id("joined-worker")
24310            .poll_timeout(Duration::from_millis(10));
24311        worker.register_workflow(
24312            "joined.workflow",
24313            |_ctx, _input| async move { Ok(Value::Null) },
24314        );
24315        worker.register_activity(
24316            "joined.activity",
24317            |_ctx, _input| async move { Ok(Value::Null) },
24318        );
24319        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
24320            Ok(Value::Null)
24321        });
24322
24323        worker
24324            .run_until(tokio::time::sleep(Duration::from_millis(20)))
24325            .await
24326            .expect("normal shutdown");
24327
24328        let deregistration_path = "/api/worker/registrations/mock-worker";
24329        assert_eq!(server.request_count(deregistration_path), 1);
24330        for poll_path in [
24331            "/api/worker/workflow-tasks/poll",
24332            "/api/worker/activity-tasks/poll",
24333            "/api/worker/query-tasks/poll",
24334        ] {
24335            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
24336        }
24337        assert_eq!(
24338            server.captured_paths().last().map(String::as_str),
24339            Some(deregistration_path),
24340            "deregistration must start only after every poller has joined"
24341        );
24342    }
24343
24344    #[tokio::test]
24345    async fn registration_failure_does_not_deregister() {
24346        let server = MockWorkerServer::rejected_registration();
24347        let client = Client::builder(server.base_url())
24348            .timeout(Duration::from_secs(2))
24349            .build()
24350            .expect("client");
24351        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
24352
24353        let error = worker
24354            .run_until(async {})
24355            .await
24356            .expect_err("registration must fail");
24357        assert!(matches!(
24358            error,
24359            Error::Http {
24360                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
24361                ..
24362            }
24363        ));
24364        assert!(server
24365            .captured_paths()
24366            .iter()
24367            .all(|path| !path.starts_with("/api/worker/registrations/")));
24368    }
24369
24370    #[tokio::test]
24371    async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
24372        let server = MockWorkerServer::rejected_registration_protocol();
24373        let client = Client::builder(server.base_url())
24374            .timeout(Duration::from_secs(2))
24375            .build()
24376            .expect("client");
24377        let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
24378
24379        let error = worker
24380            .run_until(async {})
24381            .await
24382            .expect_err("a protocol 1.16 server must reject this worker");
24383        let Error::Protocol(failure) = error else {
24384            panic!("expected typed protocol rejection");
24385        };
24386        assert_eq!(failure.reason, "unsupported_protocol_version");
24387        assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
24388        assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
24389        assert_eq!(
24390            server
24391                .worker_protocol_for("/api/worker/register")
24392                .as_deref(),
24393            Some(WORKER_PROTOCOL_VERSION)
24394        );
24395    }
24396
24397    #[tokio::test]
24398    async fn declined_registration_does_not_deregister() {
24399        let server = MockWorkerServer::declined_registration();
24400        let client = Client::builder(server.base_url())
24401            .timeout(Duration::from_secs(2))
24402            .build()
24403            .expect("client");
24404        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
24405
24406        let error = worker
24407            .run_until(async {})
24408            .await
24409            .expect_err("declined registration must fail");
24410        assert!(matches!(error, Error::WorkerLoop(_)));
24411        assert!(error.to_string().contains("was not accepted"));
24412        assert!(server
24413            .captured_paths()
24414            .iter()
24415            .all(|path| !path.starts_with("/api/worker/registrations/")));
24416    }
24417
24418    #[tokio::test]
24419    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
24420        let server = MockWorkerServer::rejected_deregistration();
24421        let client = Client::builder(server.base_url())
24422            .timeout(Duration::from_secs(2))
24423            .build()
24424            .expect("client");
24425        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
24426
24427        let error = worker
24428            .run_until(async {})
24429            .await
24430            .expect_err("deregistration must fail");
24431        assert!(matches!(
24432            error,
24433            Error::Http {
24434                status: reqwest::StatusCode::FORBIDDEN,
24435                ..
24436            }
24437        ));
24438        assert_eq!(
24439            server.request_count("/api/worker/registrations/mock-worker"),
24440            1
24441        );
24442    }
24443
24444    #[tokio::test]
24445    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
24446        let server = MockWorkerServer::rejected_deregistration_protocol();
24447        let client = Client::builder(server.base_url())
24448            .timeout(Duration::from_secs(2))
24449            .build()
24450            .expect("client");
24451        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
24452
24453        let error = worker
24454            .run_until(async {})
24455            .await
24456            .expect_err("protocol rejection must fail shutdown");
24457        let Error::Protocol(failure) = error else {
24458            panic!("expected typed protocol failure");
24459        };
24460        assert_eq!(failure.reason, "unsupported_protocol_version");
24461        assert_eq!(
24462            failure.requested_version.as_deref(),
24463            Some(WORKER_PROTOCOL_VERSION)
24464        );
24465        assert_eq!(
24466            server.request_count("/api/worker/registrations/mock-worker"),
24467            1
24468        );
24469    }
24470
24471    #[tokio::test]
24472    async fn primary_poller_error_retains_deregistration_failure_context() {
24473        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
24474        let client = Client::builder(server.base_url())
24475            .timeout(Duration::from_secs(2))
24476            .build()
24477            .expect("client");
24478        let mut worker = Worker::new(client, "rust-workers")
24479            .worker_id("combined-failure")
24480            .poll_timeout(Duration::from_millis(10));
24481        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
24482            Ok(Value::Null)
24483        });
24484
24485        let error = worker
24486            .run()
24487            .await
24488            .expect_err("worker and cleanup must fail");
24489        let summary = error.to_string();
24490        assert!(summary.contains("authentication_failed"));
24491        assert!(summary.contains("worker cannot deregister"));
24492        let Error::WorkerShutdown {
24493            primary,
24494            deregistration,
24495        } = error
24496        else {
24497            panic!("expected combined worker shutdown error");
24498        };
24499        assert!(matches!(
24500            *primary,
24501            Error::Http {
24502                status: reqwest::StatusCode::UNAUTHORIZED,
24503                ..
24504            }
24505        ));
24506        assert!(matches!(
24507            *deregistration,
24508            Error::Http {
24509                status: reqwest::StatusCode::FORBIDDEN,
24510                ..
24511            }
24512        ));
24513        assert_eq!(
24514            server.request_count("/api/worker/registrations/mock-worker"),
24515            1
24516        );
24517    }
24518
24519    #[tokio::test]
24520    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
24521        let server = MockWorkerServer::start();
24522        let client = Client::builder(server.base_url())
24523            .timeout(Duration::from_secs(2))
24524            .build()
24525            .expect("client");
24526        let mut worker = Worker::new(client, "rust-workers")
24527            .worker_id("activity-only-worker")
24528            .poll_timeout(Duration::from_millis(10));
24529
24530        worker.register_activity(
24531            "activity.only",
24532            |_ctx, _args| async move { Ok(Value::Null) },
24533        );
24534
24535        worker.run_until(async {}).await.expect("run worker");
24536    }
24537
24538    #[tokio::test]
24539    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
24540        let server = MockWorkerServer::start();
24541        let client = Client::builder(server.base_url())
24542            .timeout(Duration::from_secs(2))
24543            .build()
24544            .expect("client");
24545        let mut worker = Worker::new(client, "rust-workers")
24546            .worker_id("workflow-only-worker")
24547            .poll_timeout(Duration::from_millis(10));
24548
24549        worker.register_workflow(
24550            "workflow.only",
24551            |_ctx, _input| async move { Ok(Value::Null) },
24552        );
24553
24554        worker.run_until(async {}).await.expect("run worker");
24555    }
24556
24557    #[tokio::test]
24558    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
24559        let server = MockWorkerServer::start();
24560        let client = Client::builder(server.base_url())
24561            .timeout(Duration::from_secs(2))
24562            .build()
24563            .expect("client");
24564        let observations = Arc::new(Mutex::new(Vec::new()));
24565        let observed = Arc::clone(&observations);
24566        let mut worker = Worker::new(client, "rust-workers")
24567            .worker_id("observed-heartbeat-worker")
24568            .poll_timeout(Duration::from_millis(10))
24569            .on_worker_heartbeat(move |observation| {
24570                observed
24571                    .lock()
24572                    .expect("heartbeat observations")
24573                    .push(observation.clone());
24574            });
24575
24576        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
24577            Ok(Value::Null)
24578        });
24579        let acknowledged = Arc::clone(&observations);
24580        worker
24581            .run_until(async move {
24582                tokio::time::timeout(Duration::from_secs(2), async move {
24583                    loop {
24584                        if !acknowledged
24585                            .lock()
24586                            .expect("heartbeat observations")
24587                            .is_empty()
24588                        {
24589                            break;
24590                        }
24591                        tokio::time::sleep(Duration::from_millis(1)).await;
24592                    }
24593                })
24594                .await
24595                .expect("heartbeat acknowledgement within timeout");
24596            })
24597            .await
24598            .expect("run worker");
24599
24600        let observations = observations.lock().expect("heartbeat observations");
24601        let first = observations.first().expect("heartbeat acknowledgement");
24602        assert_eq!(first.worker_id, "observed-heartbeat-worker");
24603        assert_eq!(first.task_queue, "rust-workers");
24604        assert!(first.acknowledged_at_unix_millis > 0);
24605        assert_eq!(first.acknowledgement, json!({}));
24606    }
24607
24608    #[tokio::test]
24609    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
24610        let server = MockWorkerServer::delayed_heartbeat_worker();
24611        let client = Client::builder(server.base_url())
24612            .timeout(Duration::from_secs(3))
24613            .build()
24614            .expect("client");
24615        let observations = Arc::new(Mutex::new(Vec::new()));
24616        let observed = Arc::clone(&observations);
24617        let mut worker = Worker::new(client, "rust-snapshot-workers")
24618            .worker_id("rust-snapshot-worker")
24619            .poll_timeout(Duration::from_millis(10))
24620            .on_worker_heartbeat(move |observation| {
24621                observed
24622                    .lock()
24623                    .expect("heartbeat observations")
24624                    .push(observation.clone());
24625            });
24626
24627        worker.register_workflow("snapshot", |ctx, _input| async move {
24628            ctx.wait_signal("finish").await?;
24629            Ok(json!({"status": "finished"}))
24630        });
24631        worker.register_query("snapshot", "current", |ctx, _args| async move {
24632            Ok(json!(ctx
24633                .signals("increment")
24634                .iter()
24635                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24636                .sum::<i64>()))
24637        });
24638        worker.register_activity("cancel-aware", |_ctx, _args| async move {
24639            Ok(json!({"late": "completion"}))
24640        });
24641
24642        worker
24643            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
24644            .await
24645            .expect("delayed heartbeat must allow a clean worker shutdown");
24646
24647        let observations = observations.lock().expect("heartbeat observations");
24648        assert!(
24649            observations.len() >= 3,
24650            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
24651        );
24652        assert!(
24653            observations.windows(2).all(|pair| {
24654                pair[1].acknowledged_at_unix_millis
24655                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
24656                    >= 850
24657            }),
24658            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
24659        );
24660        drop(observations);
24661
24662        let heartbeat_times = server.request_times("/api/worker/heartbeat");
24663        let delayed_request_at = *heartbeat_times
24664            .get(1)
24665            .expect("intentionally delayed heartbeat request");
24666        let delay_window_start = delayed_request_at + Duration::from_millis(100);
24667        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
24668        for path in [
24669            "/api/worker/workflow-tasks/poll",
24670            "/api/worker/activity-tasks/poll",
24671            "/api/worker/query-tasks/poll",
24672        ] {
24673            assert!(
24674                server
24675                    .request_times(path)
24676                    .iter()
24677                    .any(|received_at| *received_at >= delay_window_start
24678                        && *received_at <= delay_window_end),
24679                "{path} must keep polling while a heartbeat acknowledgement is delayed"
24680            );
24681        }
24682        assert!(
24683            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
24684            "workflow work must be settled"
24685        );
24686        assert!(
24687            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
24688            "activity work must be settled"
24689        );
24690        assert!(
24691            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
24692            "query work must be settled"
24693        );
24694    }
24695
24696    #[tokio::test]
24697    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
24698        let server = MockWorkerServer::heartbeat_retry_worker();
24699        let client = Client::builder(server.base_url())
24700            .timeout(Duration::from_secs(2))
24701            .build()
24702            .expect("client");
24703        let observations = Arc::new(Mutex::new(Vec::new()));
24704        let observed = Arc::clone(&observations);
24705        let worker = Worker::new(client, "rust-workers")
24706            .worker_id("heartbeat-retry-worker")
24707            .retry_policy(WorkerRetryPolicy {
24708                max_retries: 1,
24709                initial_backoff: Duration::from_millis(300),
24710                max_backoff: Duration::from_millis(300),
24711            })
24712            .on_worker_heartbeat(move |observation| {
24713                observed
24714                    .lock()
24715                    .expect("heartbeat observations")
24716                    .push(observation.clone());
24717            });
24718
24719        worker
24720            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
24721            .await
24722            .expect("retryable heartbeat failure must remain bounded and recover");
24723
24724        let observations = observations.lock().expect("heartbeat observations");
24725        assert!(observations.len() >= 3, "heartbeat retry must recover");
24726        assert!(
24727            observations.windows(2).all(|pair| {
24728                pair[1]
24729                    .acknowledged_at_unix_millis
24730                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
24731                    >= 850
24732            }),
24733            "a successful retry must start a fresh advertised cadence: {observations:?}"
24734        );
24735        assert_eq!(
24736            server.request_count("/api/worker/heartbeat"),
24737            observations.len() + 1,
24738            "one retryable failure must add exactly one bounded request"
24739        );
24740    }
24741
24742    #[tokio::test]
24743    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
24744        let server = MockWorkerServer::waiting_query_worker();
24745        let client = Client::builder(server.base_url())
24746            .timeout(Duration::from_secs(2))
24747            .build()
24748            .expect("client");
24749        let observations = Arc::new(Mutex::new(Vec::new()));
24750        let observed = Arc::clone(&observations);
24751        let mut worker = Worker::new(client, "rust-snapshot-workers")
24752            .worker_id("rust-snapshot-worker")
24753            .poll_timeout(Duration::from_millis(10))
24754            .on_worker_heartbeat(move |observation| {
24755                observed
24756                    .lock()
24757                    .expect("heartbeat observations")
24758                    .push(observation.clone());
24759            });
24760
24761        worker.register_workflow("snapshot", |ctx, _input| async move {
24762            ctx.wait_signal("finish").await?;
24763            Ok(json!({"status": "finished"}))
24764        });
24765        worker.register_query("snapshot", "current", |ctx, _args| async move {
24766            let current = ctx
24767                .signals("increment")
24768                .iter()
24769                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24770                .sum::<i64>();
24771            Ok(json!(current))
24772        });
24773        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24774
24775        worker
24776            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24777            .await
24778            .expect("pending workflow and query poller must remain live until shutdown");
24779
24780        assert!(
24781            observations.lock().expect("heartbeat observations").len() >= 4,
24782            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24783        );
24784        assert!(
24785            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24786            "workflow polling must continue after empty replay acknowledgements"
24787        );
24788        assert!(
24789            server.request_count("/api/worker/query-tasks/poll") >= 2,
24790            "query polling must continue after serving the current query"
24791        );
24792        assert_eq!(
24793            server.request_body("/api/worker/register")["capabilities"],
24794            json!([
24795                CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24796                DURABLE_SELECTION_CAPABILITY,
24797                MEMO_UPSERTS_CAPABILITY,
24798                TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24799                QUERY_TASKS_CAPABILITY,
24800                WORKFLOW_UPDATES_CAPABILITY,
24801                MESSAGE_STREAMS_CAPABILITY
24802            ])
24803        );
24804        assert_eq!(
24805            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24806            json!({
24807                "queries": ["current"],
24808                "query_contracts": [],
24809                "signals": [],
24810                "signal_contracts": [],
24811                "updates": ["replace"],
24812                "update_contracts": [],
24813                "update_validators": [],
24814            })
24815        );
24816
24817        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24818        assert_eq!(
24819            opened["commands"],
24820            json!([{
24821                "type": "open_signal_wait",
24822                "signal_name": "finish",
24823            }])
24824        );
24825
24826        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24827            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24828            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24829            let failure = server.request_body(&fail_path);
24830            assert_eq!(
24831                failure["failure"]["type"],
24832                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24833            );
24834            assert_eq!(server.request_count(&completion_path), 0);
24835        }
24836
24837        let query_completion =
24838            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24839        assert_eq!(query_completion["result"], json!(8));
24840
24841        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24842        assert_eq!(
24843            server.request_count(terminal_path),
24844            1,
24845            "the matching signal must settle the workflow exactly once"
24846        );
24847        let terminal = server.request_body(terminal_path);
24848        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24849        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24850        assert_eq!(
24851            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24852                .expect("terminal workflow result"),
24853            json!({"status": "finished"})
24854        );
24855    }
24856
24857    #[tokio::test]
24858    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24859        let server = MockWorkerServer::transient_worker_failures();
24860        let client = Client::builder(server.base_url())
24861            .timeout(Duration::from_secs(2))
24862            .build()
24863            .expect("client");
24864        let mut worker = Worker::new(client, "rust-workers")
24865            .worker_id("retry-worker")
24866            .poll_timeout(Duration::from_millis(10))
24867            .retry_policy(WorkerRetryPolicy {
24868                max_retries: 2,
24869                initial_backoff: Duration::from_millis(1),
24870                max_backoff: Duration::from_millis(1),
24871            });
24872        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24873        worker.register_activity(
24874            "counter.activity",
24875            |_ctx, _input| async move { Ok(Value::Null) },
24876        );
24877        worker.register_query(
24878            "counter",
24879            "current",
24880            |_ctx, _args| async move { Ok(json!(8)) },
24881        );
24882
24883        worker
24884            .run_until(tokio::time::sleep(Duration::from_millis(75)))
24885            .await
24886            .expect("transient failures must not stop the worker");
24887
24888        for path in [
24889            "/api/worker/heartbeat",
24890            "/api/worker/workflow-tasks/poll",
24891            "/api/worker/activity-tasks/poll",
24892            "/api/worker/query-tasks/poll",
24893        ] {
24894            assert!(
24895                server.request_count(path) >= 2,
24896                "{path} must continue after its transient failure"
24897            );
24898        }
24899    }
24900
24901    #[tokio::test]
24902    async fn worker_continues_after_long_poll_capacity_backpressure() {
24903        let server = MockWorkerServer::capacity_limited_activity_poll();
24904        let client = Client::builder(server.base_url())
24905            .timeout(Duration::from_secs(2))
24906            .build()
24907            .expect("client");
24908        let mut worker = Worker::new(client, "rust-workers")
24909            .worker_id("capacity-worker")
24910            .poll_timeout(Duration::from_millis(10))
24911            .retry_policy(WorkerRetryPolicy {
24912                max_retries: 0,
24913                initial_backoff: Duration::from_millis(1),
24914                max_backoff: Duration::from_millis(1),
24915            });
24916        worker.register_activity("capacity.activity", |_ctx, _input| async move {
24917            Ok(json!({"handled": true}))
24918        });
24919
24920        worker
24921            .run_until(tokio::time::sleep(Duration::from_millis(50)))
24922            .await
24923            .expect("capacity backpressure must not stop the worker");
24924
24925        assert!(
24926            server.request_count("/api/worker/activity-tasks/poll") >= 2,
24927            "the activity poller must continue after capacity backpressure"
24928        );
24929        assert_eq!(
24930            server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24931            1,
24932            "the worker must complete work returned after capacity recovers"
24933        );
24934    }
24935
24936    #[test]
24937    fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
24938        let capacity = Error::Http {
24939            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24940            body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
24941        };
24942        assert_eq!(
24943            worker_poll_capacity_retry_after(&capacity),
24944            Some(Duration::from_secs(3))
24945        );
24946
24947        let rejected_capacity = Error::Http {
24948            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24949            body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
24950        };
24951        assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
24952        assert!(!worker_operation_is_retryable(&rejected_capacity));
24953
24954        let ordinary_rate_limit = Error::Http {
24955            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24956            body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
24957                .to_string(),
24958        };
24959        assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
24960        assert!(worker_operation_is_retryable(&ordinary_rate_limit));
24961    }
24962
24963    #[tokio::test]
24964    async fn worker_bounds_transport_retries() {
24965        let server = MockWorkerServer::unavailable_polls();
24966        let client = Client::builder(server.base_url())
24967            .timeout(Duration::from_secs(2))
24968            .build()
24969            .expect("client");
24970        let mut worker = Worker::new(client, "rust-workers")
24971            .worker_id("bounded-retry-worker")
24972            .poll_timeout(Duration::from_millis(10))
24973            .retry_policy(WorkerRetryPolicy {
24974                max_retries: 2,
24975                initial_backoff: Duration::from_millis(1),
24976                max_backoff: Duration::from_millis(1),
24977            });
24978        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24979
24980        let error = worker.run().await.expect_err("retry bound must terminate");
24981        assert!(matches!(error, Error::Transport(_)));
24982        assert_eq!(
24983            server.request_count("/api/worker/workflow-tasks/poll"),
24984            3,
24985            "one initial request plus exactly two retries"
24986        );
24987    }
24988
24989    #[tokio::test]
24990    async fn worker_retry_policy_can_disable_poll_retries() {
24991        let server = MockWorkerServer::unavailable_polls();
24992        let client = Client::builder(server.base_url())
24993            .timeout(Duration::from_secs(2))
24994            .build()
24995            .expect("client");
24996        let mut worker = Worker::new(client, "rust-workers")
24997            .worker_id("no-retry-worker")
24998            .poll_timeout(Duration::from_millis(10))
24999            .retry_policy(WorkerRetryPolicy {
25000                max_retries: 0,
25001                initial_backoff: Duration::from_millis(1),
25002                max_backoff: Duration::from_millis(1),
25003            });
25004        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25005
25006        let error = worker
25007            .run_once()
25008            .await
25009            .expect_err("disabled retries must return the first transport failure");
25010        assert!(matches!(error, Error::Transport(_)));
25011        assert_eq!(
25012            server.request_count("/api/worker/workflow-tasks/poll"),
25013            1,
25014            "max_retries=0 must send only the initial request"
25015        );
25016    }
25017
25018    #[tokio::test]
25019    async fn worker_does_not_retry_authentication_failures() {
25020        let server = MockWorkerServer::unauthorized_polls();
25021        let client = Client::builder(server.base_url())
25022            .timeout(Duration::from_secs(2))
25023            .build()
25024            .expect("client");
25025        let mut worker = Worker::new(client, "rust-workers")
25026            .worker_id("unauthorized-worker")
25027            .poll_timeout(Duration::from_millis(10));
25028        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25029
25030        let error = worker
25031            .run()
25032            .await
25033            .expect_err("authentication must terminate");
25034        let Error::Http { status, body } = error else {
25035            panic!("expected stable HTTP authentication error");
25036        };
25037        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
25038        assert!(body.contains("authentication_failed"));
25039        assert_eq!(
25040            server.request_count("/api/worker/workflow-tasks/poll"),
25041            1,
25042            "authentication failures must not be retried"
25043        );
25044    }
25045
25046    #[derive(Clone, Debug)]
25047    struct CapturedRequest {
25048        headers: String,
25049        method: String,
25050        path: String,
25051        authorization: Option<String>,
25052        namespace: Option<String>,
25053        worker_protocol: Option<String>,
25054        control_protocol: Option<String>,
25055        body: String,
25056        received_at: Instant,
25057    }
25058
25059    struct MockWorkerServer {
25060        addr: SocketAddr,
25061        stop: Arc<AtomicBool>,
25062        requests: Arc<Mutex<Vec<CapturedRequest>>>,
25063        thread: Option<thread::JoinHandle<()>>,
25064    }
25065
25066    type RequestOverride = fn(&str, &str, usize) -> Option<(&'static str, String)>;
25067
25068    #[derive(Clone, Copy, Default)]
25069    struct MockWorkerBehavior {
25070        response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
25071        request_override: Option<RequestOverride>,
25072        storage_refusals: usize,
25073        storage_path: Option<&'static str>,
25074        storage_unavailable: bool,
25075        storage_mid_poll: bool,
25076        storage_activity: bool,
25077        storage_query: bool,
25078        storage_wrong_poll_id: bool,
25079        reject_query_protocol: bool,
25080        reject_query_completion: bool,
25081        waiting_query_worker: bool,
25082        decline_registration: bool,
25083        complete_named_signal: bool,
25084        poll_failures_per_path: usize,
25085        long_poll_capacity_responses_per_path: usize,
25086        heartbeat_failures: usize,
25087        heartbeat_failure_request: Option<usize>,
25088        delayed_heartbeat_request: Option<usize>,
25089        heartbeat_response_delay: Duration,
25090        concurrent_requests: bool,
25091        unauthorized_polls: bool,
25092        reject_registration: bool,
25093        reject_registration_protocol: bool,
25094        reject_deregistration: bool,
25095        reject_deregistration_protocol: bool,
25096        cancelled_activity: bool,
25097        draining_polls: bool,
25098        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
25099        workflow_completion_status: Option<&'static str>,
25100        workflow_completion_body: Option<&'static str>,
25101    }
25102
25103    impl MockWorkerServer {
25104        fn start() -> Self {
25105            Self::start_with_behavior(MockWorkerBehavior::default())
25106        }
25107
25108        fn reject_query_protocol() -> Self {
25109            Self::start_with_behavior(MockWorkerBehavior {
25110                reject_query_protocol: true,
25111                ..MockWorkerBehavior::default()
25112            })
25113        }
25114
25115        fn reject_query_completion() -> Self {
25116            Self::start_with_behavior(MockWorkerBehavior {
25117                reject_query_completion: true,
25118                ..MockWorkerBehavior::default()
25119            })
25120        }
25121
25122        fn waiting_query_worker() -> Self {
25123            Self::start_with_behavior(MockWorkerBehavior {
25124                waiting_query_worker: true,
25125                complete_named_signal: true,
25126                ..MockWorkerBehavior::default()
25127            })
25128        }
25129
25130        fn transient_worker_failures() -> Self {
25131            Self::start_with_behavior(MockWorkerBehavior {
25132                poll_failures_per_path: 1,
25133                heartbeat_failures: 1,
25134                ..MockWorkerBehavior::default()
25135            })
25136        }
25137
25138        fn consecutive_poll_failures(count: usize) -> Self {
25139            Self::start_with_behavior(MockWorkerBehavior {
25140                poll_failures_per_path: count,
25141                ..MockWorkerBehavior::default()
25142            })
25143        }
25144
25145        fn capacity_limited_activity_poll() -> Self {
25146            Self::start_with_behavior(MockWorkerBehavior {
25147                long_poll_capacity_responses_per_path: 1,
25148                ..MockWorkerBehavior::default()
25149            })
25150        }
25151
25152        fn delayed_heartbeat_worker() -> Self {
25153            Self::start_with_behavior(MockWorkerBehavior {
25154                waiting_query_worker: true,
25155                delayed_heartbeat_request: Some(2),
25156                heartbeat_response_delay: Duration::from_millis(1_500),
25157                concurrent_requests: true,
25158                cancelled_activity: true,
25159                ..MockWorkerBehavior::default()
25160            })
25161        }
25162
25163        fn heartbeat_retry_worker() -> Self {
25164            Self::start_with_behavior(MockWorkerBehavior {
25165                waiting_query_worker: true,
25166                heartbeat_failure_request: Some(2),
25167                concurrent_requests: true,
25168                ..MockWorkerBehavior::default()
25169            })
25170        }
25171
25172        fn unavailable_polls() -> Self {
25173            Self::start_with_behavior(MockWorkerBehavior {
25174                poll_failures_per_path: usize::MAX,
25175                ..MockWorkerBehavior::default()
25176            })
25177        }
25178
25179        fn unauthorized_polls() -> Self {
25180            Self::start_with_behavior(MockWorkerBehavior {
25181                unauthorized_polls: true,
25182                ..MockWorkerBehavior::default()
25183            })
25184        }
25185
25186        fn rejected_registration() -> Self {
25187            Self::start_with_behavior(MockWorkerBehavior {
25188                reject_registration: true,
25189                ..MockWorkerBehavior::default()
25190            })
25191        }
25192
25193        fn rejected_registration_protocol() -> Self {
25194            Self::start_with_behavior(MockWorkerBehavior {
25195                reject_registration_protocol: true,
25196                ..MockWorkerBehavior::default()
25197            })
25198        }
25199
25200        fn declined_registration() -> Self {
25201            Self::start_with_behavior(MockWorkerBehavior {
25202                decline_registration: true,
25203                ..MockWorkerBehavior::default()
25204            })
25205        }
25206
25207        fn rejected_deregistration() -> Self {
25208            Self::start_with_behavior(MockWorkerBehavior {
25209                reject_deregistration: true,
25210                ..MockWorkerBehavior::default()
25211            })
25212        }
25213
25214        fn rejected_deregistration_protocol() -> Self {
25215            Self::start_with_behavior(MockWorkerBehavior {
25216                reject_deregistration_protocol: true,
25217                ..MockWorkerBehavior::default()
25218            })
25219        }
25220
25221        fn unauthorized_polls_and_rejected_deregistration() -> Self {
25222            Self::start_with_behavior(MockWorkerBehavior {
25223                unauthorized_polls: true,
25224                reject_deregistration: true,
25225                ..MockWorkerBehavior::default()
25226            })
25227        }
25228
25229        fn cancelled_activity() -> Self {
25230            Self::start_with_behavior(MockWorkerBehavior {
25231                cancelled_activity: true,
25232                ..MockWorkerBehavior::default()
25233            })
25234        }
25235
25236        fn draining_polls() -> Self {
25237            Self::start_with_behavior(MockWorkerBehavior {
25238                draining_polls: true,
25239                ..MockWorkerBehavior::default()
25240            })
25241        }
25242
25243        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
25244            Self::start_with_behavior(MockWorkerBehavior {
25245                invalid_task_payload_codec: Some(codec),
25246                ..MockWorkerBehavior::default()
25247            })
25248        }
25249
25250        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
25251            Self::start_with_behavior(MockWorkerBehavior {
25252                workflow_completion_status: Some(status),
25253                workflow_completion_body: Some(body),
25254                ..MockWorkerBehavior::default()
25255            })
25256        }
25257
25258        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
25259            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
25260            listener
25261                .set_nonblocking(true)
25262                .expect("configure mock listener");
25263            let addr = listener.local_addr().expect("mock server address");
25264            let stop = Arc::new(AtomicBool::new(false));
25265            let server_stop = Arc::clone(&stop);
25266            let requests = Arc::new(Mutex::new(Vec::new()));
25267            let server_requests = Arc::clone(&requests);
25268            let thread = thread::spawn(move || {
25269                let mut request_threads = Vec::new();
25270                while !server_stop.load(Ordering::SeqCst) {
25271                    match listener.accept() {
25272                        Ok((mut stream, _)) => {
25273                            if behavior.concurrent_requests {
25274                                let requests = Arc::clone(&server_requests);
25275                                request_threads.push(thread::spawn(move || {
25276                                    handle_mock_worker_request(&mut stream, &requests, behavior)
25277                                }));
25278                            } else {
25279                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
25280                            }
25281                        }
25282                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
25283                            let mut index = 0;
25284                            while index < request_threads.len() {
25285                                if request_threads[index].is_finished() {
25286                                    request_threads
25287                                        .swap_remove(index)
25288                                        .join()
25289                                        .expect("join mock request");
25290                                } else {
25291                                    index += 1;
25292                                }
25293                            }
25294                            thread::sleep(Duration::from_millis(5));
25295                        }
25296                        Err(_) => break,
25297                    }
25298                }
25299                for request_thread in request_threads {
25300                    request_thread.join().expect("join mock request");
25301                }
25302            });
25303
25304            Self {
25305                addr,
25306                stop,
25307                requests,
25308                thread: Some(thread),
25309            }
25310        }
25311
25312        fn base_url(&self) -> String {
25313            format!("http://{}", self.addr)
25314        }
25315
25316        fn worker_protocol_for(&self, path: &str) -> Option<String> {
25317            self.requests
25318                .lock()
25319                .expect("captured requests")
25320                .iter()
25321                .find(|request| request.path == path)
25322                .and_then(|request| request.worker_protocol.clone())
25323        }
25324
25325        fn control_protocol_for(&self, path: &str) -> Option<String> {
25326            self.requests
25327                .lock()
25328                .expect("captured requests")
25329                .iter()
25330                .find(|request| request.path == path)
25331                .and_then(|request| request.control_protocol.clone())
25332        }
25333
25334        fn method_for(&self, path: &str) -> Option<String> {
25335            self.requests
25336                .lock()
25337                .expect("captured requests")
25338                .iter()
25339                .find(|request| request.path == path)
25340                .map(|request| request.method.clone())
25341        }
25342
25343        fn authorization_for(&self, path: &str) -> Option<String> {
25344            self.requests
25345                .lock()
25346                .expect("captured requests")
25347                .iter()
25348                .find(|request| request.path == path)
25349                .and_then(|request| request.authorization.clone())
25350        }
25351
25352        fn namespace_for(&self, path: &str) -> Option<String> {
25353            self.requests
25354                .lock()
25355                .expect("captured requests")
25356                .iter()
25357                .find(|request| request.path == path)
25358                .and_then(|request| request.namespace.clone())
25359        }
25360
25361        fn request_count(&self, path: &str) -> usize {
25362            self.requests
25363                .lock()
25364                .expect("captured requests")
25365                .iter()
25366                .filter(|request| request.path == path)
25367                .count()
25368        }
25369
25370        fn captured_paths(&self) -> Vec<String> {
25371            self.requests
25372                .lock()
25373                .expect("captured requests")
25374                .iter()
25375                .map(|request| request.path.clone())
25376                .collect()
25377        }
25378
25379        fn request_times(&self, path: &str) -> Vec<Instant> {
25380            self.requests
25381                .lock()
25382                .expect("captured requests")
25383                .iter()
25384                .filter(|request| request.path == path)
25385                .map(|request| request.received_at)
25386                .collect()
25387        }
25388
25389        fn request_body(&self, path: &str) -> Value {
25390            let requests = self.requests.lock().expect("captured requests");
25391            let body = &requests
25392                .iter()
25393                .find(|request| request.path == path)
25394                .unwrap_or_else(|| panic!("missing request for {path}"))
25395                .body;
25396            serde_json::from_str(body).unwrap_or_else(|error| {
25397                panic!("invalid JSON request body for {path}: {error}: {body:?}")
25398            })
25399        }
25400
25401        fn request_bodies(&self, path: &str) -> Vec<Value> {
25402            self.requests
25403                .lock()
25404                .expect("captured requests")
25405                .iter()
25406                .filter(|request| request.path == path)
25407                .map(|request| {
25408                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
25409                        panic!(
25410                            "invalid JSON request body for {path}: {error}: {:?}",
25411                            request.body
25412                        )
25413                    })
25414                })
25415                .collect()
25416        }
25417    }
25418
25419    impl Drop for MockWorkerServer {
25420        fn drop(&mut self) {
25421            self.stop.store(true, Ordering::SeqCst);
25422            let _ = TcpStream::connect(self.addr);
25423
25424            if let Some(thread) = self.thread.take() {
25425                thread.join().expect("join mock server");
25426            }
25427        }
25428    }
25429
25430    fn handle_mock_worker_request(
25431        stream: &mut TcpStream,
25432        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
25433        behavior: MockWorkerBehavior,
25434    ) {
25435        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
25436        let mut buffer = [0_u8; 8192];
25437        let mut request = Vec::new();
25438
25439        loop {
25440            match stream.read(&mut buffer) {
25441                Ok(0) => break,
25442                Ok(read) => {
25443                    request.extend_from_slice(&buffer[..read]);
25444                    if mock_request_is_complete(&request) {
25445                        break;
25446                    }
25447                }
25448                Err(error)
25449                    if matches!(
25450                        error.kind(),
25451                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
25452                    ) =>
25453                {
25454                    break;
25455                }
25456                Err(_) => return,
25457            }
25458        }
25459
25460        let request = String::from_utf8_lossy(&request);
25461        let body = request
25462            .split_once("\r\n\r\n")
25463            .map(|(_, body)| body)
25464            .unwrap_or_default();
25465        let path = request
25466            .lines()
25467            .next()
25468            .and_then(|line| line.split_whitespace().nth(1))
25469            .unwrap_or_default();
25470        let method = request
25471            .lines()
25472            .next()
25473            .and_then(|line| line.split_whitespace().next())
25474            .unwrap_or_default();
25475        let authorization = request.lines().find_map(|line| {
25476            let (name, value) = line.split_once(':')?;
25477            name.eq_ignore_ascii_case("Authorization")
25478                .then(|| value.trim().to_string())
25479        });
25480        let namespace = request.lines().find_map(|line| {
25481            let (name, value) = line.split_once(':')?;
25482            name.eq_ignore_ascii_case("X-Namespace")
25483                .then(|| value.trim().to_string())
25484        });
25485        let worker_protocol = request.lines().find_map(|line| {
25486            let (name, value) = line.split_once(':')?;
25487            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
25488                .then(|| value.trim().to_string())
25489        });
25490        let control_protocol = request.lines().find_map(|line| {
25491            let (name, value) = line.split_once(':')?;
25492            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
25493                .then(|| value.trim().to_string())
25494        });
25495        let request_number = {
25496            let mut requests = requests.lock().expect("captured requests");
25497            requests.push(CapturedRequest {
25498                headers: request
25499                    .split_once("\r\n\r\n")
25500                    .map_or("", |(headers, _)| headers)
25501                    .to_owned(),
25502                method: method.to_string(),
25503                path: path.to_string(),
25504                authorization,
25505                namespace,
25506                worker_protocol: worker_protocol.clone(),
25507                control_protocol,
25508                body: body.to_string(),
25509                received_at: Instant::now(),
25510            });
25511            requests
25512                .iter()
25513                .filter(|request| request.path == path)
25514                .count()
25515        };
25516
25517        if let Some(response) = behavior
25518            .request_override
25519            .and_then(|handler| handler(path, body, request_number))
25520        {
25521            write_mock_response(stream, response.0, &response.1);
25522            return;
25523        }
25524        if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
25525            write_mock_response(stream, response.0, &response.1);
25526            return;
25527        }
25528        if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
25529            return;
25530        }
25531        let pressure_path = behavior
25532            .storage_path
25533            .is_some_and(|part| path.contains(part));
25534        let prior_failures = if path.ends_with("/poll") {
25535            behavior.poll_failures_per_path
25536        } else {
25537            0
25538        };
25539        if pressure_path
25540            && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
25541        {
25542            let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
25543            let poll_id = path
25544                .ends_with("/poll")
25545                .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
25546            let mut refusal = storage_refusal(
25547                poll_id,
25548                behavior.storage_unavailable,
25549                behavior.storage_mid_poll,
25550            );
25551            if behavior.storage_wrong_poll_id {
25552                refusal["poll_request_id"] = json!("wrong-poll");
25553            }
25554            write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
25555            return;
25556        }
25557        if path.contains("/storage-task/")
25558            || path.contains("/storage-activity/")
25559            || path.contains("/storage-query/")
25560        {
25561            write_mock_response(stream, "200 OK", "{}");
25562            return;
25563        }
25564        if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
25565            write_mock_response(stream, "200 OK", &json!({"task":{
25566                "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
25567                "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
25568                "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25569                "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25570                "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
25571            }}).to_string());
25572            return;
25573        }
25574        if behavior.storage_activity
25575            && path == "/api/worker/activity-tasks/poll"
25576            && request_number == 1
25577        {
25578            write_mock_response(stream, "200 OK", &json!({"task":{
25579                "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
25580                "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25581                "attempt_number":7, "lease_owner":"storage-worker"
25582            }}).to_string());
25583            return;
25584        }
25585
25586        if path == "/api/worker/register" {
25587            if behavior.reject_registration_protocol {
25588                write_mock_response(
25589                    stream,
25590                    "400 Bad Request",
25591                    r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
25592                );
25593                return;
25594            }
25595            if behavior.reject_registration {
25596                write_mock_response(
25597                    stream,
25598                    "503 Service Unavailable",
25599                    r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
25600                );
25601                return;
25602            }
25603        }
25604
25605        if path.starts_with("/api/worker/registrations/") {
25606            if behavior.reject_deregistration_protocol {
25607                write_mock_response(
25608                    stream,
25609                    "400 Bad Request",
25610                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
25611                );
25612            } else if behavior.reject_deregistration {
25613                write_mock_response(
25614                    stream,
25615                    "403 Forbidden",
25616                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
25617                );
25618            } else {
25619                write_mock_response(
25620                    stream,
25621                    "200 OK",
25622                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
25623                );
25624            }
25625            return;
25626        }
25627
25628        let is_poll = matches!(
25629            path,
25630            "/api/worker/workflow-tasks/poll"
25631                | "/api/worker/activity-tasks/poll"
25632                | "/api/worker/query-tasks/poll"
25633        );
25634        if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
25635            write_mock_response(
25636                stream,
25637                "429 Too Many Requests",
25638                r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
25639            );
25640            return;
25641        }
25642        if is_poll && request_number <= behavior.poll_failures_per_path {
25643            return;
25644        }
25645        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
25646            return;
25647        }
25648        if path == "/api/worker/heartbeat"
25649            && behavior.heartbeat_failure_request == Some(request_number)
25650        {
25651            return;
25652        }
25653        if path == "/api/worker/heartbeat"
25654            && behavior.delayed_heartbeat_request == Some(request_number)
25655        {
25656            thread::sleep(behavior.heartbeat_response_delay);
25657        }
25658        if behavior.unauthorized_polls && is_poll {
25659            write_mock_response(
25660                stream,
25661                "401 Unauthorized",
25662                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
25663            );
25664            return;
25665        }
25666        if behavior.draining_polls && is_poll {
25667            write_mock_response(
25668                stream,
25669                "409 Conflict",
25670                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
25671            );
25672            return;
25673        }
25674
25675        if let Some(codec_case) = behavior.invalid_task_payload_codec {
25676            if is_poll && request_number == 1 {
25677                let mut task = match path {
25678                    "/api/worker/workflow-tasks/poll" => json!({
25679                        "task_id": "codec-workflow",
25680                        "workflow_type": "codec.workflow",
25681                        "payload_codec": DEFAULT_CODEC,
25682                        "workflow_task_attempt": 1,
25683                        "lease_owner": "codec-worker"
25684                    }),
25685                    "/api/worker/activity-tasks/poll" => json!({
25686                        "task_id": "codec-activity",
25687                        "activity_attempt_id": "codec-activity-attempt",
25688                        "activity_type": "codec.activity",
25689                        "payload_codec": DEFAULT_CODEC,
25690                        "attempt_number": 1,
25691                        "lease_owner": "codec-worker"
25692                    }),
25693                    "/api/worker/query-tasks/poll" => json!({
25694                        "query_task_id": "codec-query",
25695                        "query_task_attempt": 1,
25696                        "workflow_type": "codec.workflow",
25697                        "query_name": "known",
25698                        "payload_codec": DEFAULT_CODEC,
25699                        "lease_owner": "codec-worker"
25700                    }),
25701                    _ => unreachable!("is_poll limits task codec probe paths"),
25702                };
25703                codec_case.apply(&mut task);
25704                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
25705                return;
25706            }
25707
25708            if matches!(
25709                path,
25710                "/api/worker/workflow-tasks/codec-workflow/fail"
25711                    | "/api/worker/activity-tasks/codec-activity/fail"
25712                    | "/api/worker/query-tasks/codec-query/fail"
25713            ) {
25714                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
25715                return;
25716            }
25717        }
25718
25719        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
25720            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
25721            let body = format!(
25722                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
25723            );
25724            write_mock_response(stream, "400 Bad Request", &body);
25725            return;
25726        }
25727
25728        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
25729        {
25730            write_mock_response(
25731                stream,
25732                "409 Conflict",
25733                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
25734            );
25735            return;
25736        }
25737
25738        if behavior.workflow_completion_status.is_some()
25739            && path == "/api/worker/workflow-tasks/poll"
25740            && request_number == 1
25741        {
25742            write_mock_response(
25743                stream,
25744                "200 OK",
25745                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
25746            );
25747            return;
25748        }
25749
25750        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
25751            if let (Some(status), Some(body)) = (
25752                behavior.workflow_completion_status,
25753                behavior.workflow_completion_body,
25754            ) {
25755                write_mock_response(stream, status, body);
25756                return;
25757            }
25758        }
25759
25760        if behavior.waiting_query_worker {
25761            if behavior.complete_named_signal
25762                && path == "/api/worker/workflow-tasks/poll"
25763                && request_number == 1
25764            {
25765                let body = json!({
25766                    "task": {
25767                        "task_id": "snapshot-open",
25768                        "workflow_id": "snapshot-1",
25769                        "run_id": "snapshot-run-1",
25770                        "workflow_type": "snapshot",
25771                        "payload_codec": DEFAULT_CODEC,
25772                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25773                            .expect("Avro workflow arguments"),
25774                        "history_events": [],
25775                        "workflow_task_attempt": 1,
25776                        "lease_owner": "rust-snapshot-worker"
25777                    }
25778                })
25779                .to_string();
25780                write_mock_response(stream, "200 OK", &body);
25781                return;
25782            }
25783
25784            let signal_request = request_number - usize::from(behavior.complete_named_signal);
25785            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25786            if path == "/api/worker/workflow-tasks/poll"
25787                && signal_request >= 1
25788                && signal_request <= signal_request_limit
25789            {
25790                let finish = behavior.complete_named_signal && signal_request == 3;
25791                let amounts = if signal_request == 1 {
25792                    vec![3]
25793                } else {
25794                    vec![3, 5]
25795                };
25796                let task_id = if signal_request == 1 {
25797                    "snapshot-wait-3"
25798                } else if finish {
25799                    "snapshot-finish"
25800                } else {
25801                    "snapshot-wait-5"
25802                };
25803                let mut history_events = std::iter::once(json!({
25804                    "event_type": "SignalWaitOpened",
25805                    "payload": {"sequence": 1, "signal_name": "finish"}
25806                }))
25807                .chain(amounts.iter().enumerate().map(|(index, amount)| {
25808                    json!({
25809                        "event_type": "SignalReceived",
25810                        "payload": {
25811                            "signal_id": format!("increment-{amount}"),
25812                            "signal_name": "increment",
25813                            "workflow_sequence": index + 2,
25814                            "payload_codec": DEFAULT_CODEC,
25815                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25816                                .expect("Avro signal envelope")
25817                        }
25818                    })
25819                }))
25820                .collect::<Vec<_>>();
25821                let (resume_id, resume_name, resume_arguments) = if finish {
25822                    history_events.push(json!({
25823                        "event_type": "SignalReceived",
25824                        "payload": {
25825                            "signal_id": "finish",
25826                            "signal_name": "finish",
25827                            "workflow_sequence": 4,
25828                            "payload_codec": DEFAULT_CODEC,
25829                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25830                                .expect("Avro finish signal envelope")
25831                        }
25832                    }));
25833                    (
25834                        "finish".to_string(),
25835                        "finish".to_string(),
25836                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
25837                            .expect("Avro finish resume signal"),
25838                    )
25839                } else {
25840                    let amount = amounts.last().expect("amount");
25841                    (
25842                        format!("increment-{amount}"),
25843                        "increment".to_string(),
25844                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25845                            .expect("Avro increment resume signal"),
25846                    )
25847                };
25848                let body = json!({
25849                    "task": {
25850                        "task_id": task_id,
25851                        "workflow_id": "snapshot-1",
25852                        "run_id": "snapshot-run-1",
25853                        "workflow_type": "snapshot",
25854                        "payload_codec": DEFAULT_CODEC,
25855                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25856                            .expect("Avro workflow arguments"),
25857                        "history_events": history_events,
25858                        "workflow_task_attempt": 1,
25859                        "workflow_signal_id": resume_id,
25860                        "signal_name": resume_name,
25861                        "signal_arguments": resume_arguments,
25862                        "lease_owner": "rust-snapshot-worker"
25863                    }
25864                })
25865                .to_string();
25866                write_mock_response(stream, "200 OK", &body);
25867                return;
25868            }
25869
25870            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25871                let history_events = [3, 5]
25872                    .into_iter()
25873                    .enumerate()
25874                    .map(|(index, amount)| {
25875                        json!({
25876                            "event_type": "SignalReceived",
25877                            "payload": {
25878                                "signal_id": format!("increment-{amount}"),
25879                                "signal_name": "increment",
25880                                "workflow_sequence": index + 2,
25881                                "payload_codec": DEFAULT_CODEC,
25882                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25883                                    .expect("Avro query signal envelope")
25884                            }
25885                        })
25886                    })
25887                    .collect::<Vec<_>>();
25888                let body = json!({
25889                    "task": {
25890                        "query_task_id": "snapshot-current",
25891                        "query_task_attempt": 1,
25892                        "lease_owner": "rust-snapshot-worker",
25893                        "workflow_id": "snapshot-1",
25894                        "run_id": "snapshot-run-1",
25895                        "workflow_type": "snapshot",
25896                        "query_name": "current",
25897                        "payload_codec": DEFAULT_CODEC,
25898                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25899                            .expect("Avro workflow arguments"),
25900                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25901                            .expect("Avro query arguments"),
25902                        "history_events": history_events,
25903                        "run_status": "waiting"
25904                    }
25905                })
25906                .to_string();
25907                write_mock_response(stream, "200 OK", &body);
25908                return;
25909            }
25910
25911            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25912                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25913            {
25914                write_mock_response(
25915                    stream,
25916                    "200 OK",
25917                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
25918                );
25919                return;
25920            }
25921
25922            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25923                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25924                return;
25925            }
25926
25927            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25928                write_mock_response(
25929                    stream,
25930                    "200 OK",
25931                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25932                );
25933                return;
25934            }
25935
25936            if path == "/api/worker/query-tasks/snapshot-current/complete" {
25937                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
25938                return;
25939            }
25940        }
25941
25942        if matches!(
25943            path,
25944            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
25945        ) {
25946            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25947                .expect("typed mock result");
25948            let body = json!({
25949                "result": typed_fidelity_probe().into_json().expect("result projection"),
25950                "result_envelope": result,
25951            })
25952            .to_string();
25953            write_mock_response(stream, "200 OK", &body);
25954            return;
25955        }
25956
25957        if path == "/api/workflows/typed-1" {
25958            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25959                .expect("typed mock result");
25960            let body = json!({
25961                "workflow_id": "typed-1",
25962                "run_id": "run-typed-1",
25963                "workflow_type": "typed.echo",
25964                "status": "completed",
25965                "output": typed_fidelity_probe().into_json().expect("output projection"),
25966                "output_envelope": result,
25967            })
25968            .to_string();
25969            write_mock_response(stream, "200 OK", &body);
25970            return;
25971        }
25972
25973        let (status, body) = match path {
25974            "/api/cluster/info" => ("200 OK", r#"{"limits":{"max_payload_bytes":2097152}}"#),
25975            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
25976            "/api/workflows" => (
25977                "201 Created",
25978                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
25979            ),
25980            "/api/worker/register" if behavior.decline_registration => (
25981                "200 OK",
25982                r#"{"worker_id":"declined-worker","registered":false}"#,
25983            ),
25984            "/api/worker/register" if behavior.waiting_query_worker => (
25985                "200 OK",
25986                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
25987            ),
25988            "/api/worker/register" => (
25989                "200 OK",
25990                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
25991            ),
25992            "/api/worker/heartbeat" => ("200 OK", "{}"),
25993            "/api/worker/activity-tasks/poll"
25994                if behavior.cancelled_activity && request_number == 1 =>
25995            {
25996                (
25997                    "200 OK",
25998                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
25999                )
26000            }
26001            "/api/worker/activity-tasks/poll"
26002                if behavior.long_poll_capacity_responses_per_path > 0
26003                    && request_number
26004                        == behavior
26005                            .long_poll_capacity_responses_per_path
26006                            .saturating_add(1) =>
26007            {
26008                (
26009                    "200 OK",
26010                    r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
26011                )
26012            }
26013            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
26014                ("200 OK", r#"{"task":null}"#)
26015            }
26016            "/api/worker/query-tasks/poll"
26017                if behavior.reject_query_completion && request_number == 1 =>
26018            {
26019                (
26020                    "200 OK",
26021                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
26022                )
26023            }
26024            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
26025            "/api/worker/query-tasks/query-capture/complete"
26026            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
26027            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
26028                "200 OK",
26029                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
26030            ),
26031            "/api/worker/activity-tasks/activity-cancel/complete" => (
26032                "409 Conflict",
26033                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
26034            ),
26035            "/api/worker/activity-tasks/activity-typed/complete"
26036            | "/api/worker/activity-tasks/activity-typed/fail"
26037            | "/api/worker/activity-tasks/capacity-activity/complete"
26038            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
26039            "/api/workflows/counter-1/query/current" => (
26040                "200 OK",
26041                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
26042            ),
26043            "/api/workflows/counter-1/query/missing" => (
26044                "404 Not Found",
26045                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
26046            ),
26047            "/api/workflows/wf-lifecycle/cancel" => (
26048                "200 OK",
26049                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
26050            ),
26051            "/api/workflows/wf-lifecycle/terminate" => (
26052                "200 OK",
26053                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
26054            ),
26055            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
26056                "200 OK",
26057                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
26058            ),
26059            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
26060                "200 OK",
26061                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
26062            ),
26063            "/api/workflows/wf-lifecycle/runs/run-failed/redrive" => (
26064                "202 Accepted",
26065                r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
26066            ),
26067            "/api/workflows/wf-lifecycle/runs/run-failed-existing/redrive" => (
26068                "200 OK",
26069                r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed-existing","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
26070            ),
26071            "/api/workflows/wf-lifecycle/runs/run-completed/redrive" => (
26072                "409 Conflict",
26073                r#"{"workflow_id":"wf-lifecycle","run_id":"run-completed","reason":"run_not_failed","message":"Run is not failed."}"#,
26074            ),
26075            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
26076            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
26077                "409 Conflict",
26078                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
26079            ),
26080            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
26081                "200 OK",
26082                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
26083            ),
26084            "/api/workflows/wf-cancelled" => (
26085                "200 OK",
26086                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
26087            ),
26088            "/api/workflows/wf-terminated" => (
26089                "200 OK",
26090                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
26091            ),
26092            "/api/workflows/wf-timed-out" => (
26093                "200 OK",
26094                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
26095            ),
26096            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
26097                "200 OK",
26098                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
26099            ),
26100            "/api/workflows/wf-selected" => (
26101                "200 OK",
26102                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
26103            ),
26104            "/api/workflows/wf-selected/runs/run-selected" => (
26105                "200 OK",
26106                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
26107            ),
26108            _ => ("404 Not Found", r#"{"message":"not found"}"#),
26109        };
26110        write_mock_response(stream, status, body);
26111    }
26112
26113    fn mock_request_is_complete(request: &[u8]) -> bool {
26114        let Some(header_end) = request
26115            .windows(4)
26116            .position(|window| window == b"\r\n\r\n")
26117            .map(|position| position + 4)
26118        else {
26119            return false;
26120        };
26121        let headers = String::from_utf8_lossy(&request[..header_end]);
26122        let content_length = headers.lines().find_map(|line| {
26123            let (name, value) = line.split_once(':')?;
26124            name.eq_ignore_ascii_case("content-length")
26125                .then(|| value.trim().parse::<usize>().ok())
26126                .flatten()
26127        });
26128
26129        request.len() >= header_end + content_length.unwrap_or(0)
26130    }
26131
26132    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
26133        let response = format!(
26134            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
26135            body.len()
26136        );
26137
26138        let _ = stream.write_all(response.as_bytes());
26139        let _ = stream.flush();
26140    }
26141}