Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod runtime_payloads;
4mod runtime_uploads;
5
6use std::{
7    any::{type_name, Any, TypeId},
8    collections::{BTreeMap, HashMap},
9    future::Future,
10    io::{self, Read},
11    pin::Pin,
12    sync::{
13        atomic::{AtomicBool, Ordering},
14        Arc, Mutex, OnceLock,
15    },
16    task::{Context as TaskContext, Poll},
17    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
18};
19
20use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
21use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
22use chrono::DateTime;
23use futures_util::{future::OptionFuture, task::noop_waker_ref};
24use serde::{
25    de::DeserializeOwned,
26    ser::{SerializeMap, SerializeSeq},
27    Deserialize, Deserializer, Serialize, Serializer,
28};
29pub use serde_json::{json, Value};
30use sha2::{Digest, Sha256};
31use thiserror::Error;
32pub use uuid::Uuid;
33
34pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
35/// First additive worker protocol that defines portable worker-affinity features.
36pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
37pub const CONTROL_PLANE_VERSION: &str = "2";
38pub const DEFAULT_CODEC: &str = "avro";
39pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
40/// Worker-registration capability for authored condition-wait occurrence identity.
41pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
42    "condition_wait_occurrence_identity";
43/// Worker-registration capability for portable memo upserts.
44pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
45/// Worker-registration capability for server-routed read-only queries.
46pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
47/// Worker-registration capability for canonical typed search attributes.
48pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
49/// Worker-registration capability for synchronous workflow updates.
50pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
51/// Worker-registration capability for durable named input streams.
52pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
53/// Worker-registration capability for persisted first-completion selection.
54pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
55pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
56pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
57pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
58pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
59pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
60/// First additive worker protocol that defines query-task transport.
61pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
62/// First additive worker protocol that defines typed search-attribute upserts.
63pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
64/// First additive worker protocol that defines portable memo upserts.
65pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
66/// First additive worker protocol that preserves declared search-attribute types.
67pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
68/// First additive worker protocol that defines external durable condition waits.
69pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
70/// First additive worker protocol that preserves authored condition-wait occurrences.
71pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
72/// First additive worker protocol that defines durable selection groups.
73pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
74
75pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
76    let Some((major, minor)) = version.split_once('.') else {
77        return false;
78    };
79    major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
80}
81
82fn validate_user_signal_name(signal_name: &str) -> Result<()> {
83    if signal_name == MESSAGE_STREAM_SIGNAL {
84        return Err(Error::Codec(format!(
85            "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
86        )));
87    }
88    Ok(())
89}
90
91const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
92const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
93    "Workflow task waiting for scheduled history.";
94const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
95const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
96const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
97const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
98const MAX_MEMO_ENTRIES: usize = 100;
99const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
100const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
101
102const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
103    "lease_expired",
104    "query_task_not_found",
105    "query_task_not_leased",
106    "query_task_timed_out",
107];
108
109/// Truthful service-worker manifest for features this SDK currently refuses.
110pub fn portable_worker_affinity_capability_manifest() -> Value {
111    json!({
112        "local_activities": {
113            "supported": false,
114            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
115            "reason": "rust_worker_does_not_execute_record_local_activity",
116        },
117        "worker_sessions": {
118            "supported": false,
119            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
120            "reason": "rust_worker_has_no_typed_session_lifecycle",
121        },
122        "sticky_execution": {
123            "supported": false,
124            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
125            "reason": "rust_worker_uses_complete_durable_history_replay",
126        },
127    })
128}
129
130/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
131pub const AVRO_VALUE_SCHEMA_JSON: &str =
132    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
133pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
134pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
135const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
136
137static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
138static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
139    OnceLock::new();
140
141#[derive(Clone, Copy)]
142enum RequestProtocol {
143    ControlPlane,
144    Worker(&'static str),
145}
146
147pub type Result<T> = std::result::Result<T, Error>;
148
149#[derive(Debug, Error)]
150pub enum Error {
151    #[error("transport error: {0}")]
152    Transport(#[from] reqwest::Error),
153    #[error(
154        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
155    )]
156    InvalidBaseUrl,
157    #[error("json error: {0}")]
158    Json(#[from] serde_json::Error),
159    #[error("http {status}: {body}")]
160    Http {
161        status: reqwest::StatusCode,
162        body: String,
163    },
164    #[error("codec error: {0}")]
165    Codec(String),
166    #[error(transparent)]
167    QueryFailed(QueryFailure),
168    #[error(transparent)]
169    Protocol(ProtocolFailure),
170    #[error(transparent)]
171    NonDeterministicReplay(ReplayFailure),
172    #[error(transparent)]
173    ChildWorkflowFailed(ChildWorkflowFailure),
174    #[error(transparent)]
175    ActivityFailed(ActivityFailure),
176    #[error(transparent)]
177    ParallelFailed(ParallelFailure),
178    #[error(transparent)]
179    SagaCompensationFailed(SagaCompensationFailure),
180    #[error(transparent)]
181    InvalidParallelGroup(ParallelGroupError),
182    #[error(transparent)]
183    DurableOperationCancelled(DurableOperationCancelled),
184    #[error(transparent)]
185    WorkflowCancellationRequested(WorkflowCancellationRequested),
186    #[error(transparent)]
187    WorkflowCommandRejected(WorkflowCommandRejection),
188    #[error(transparent)]
189    WorkflowFailed(WorkflowTerminalOutcome),
190    #[error(transparent)]
191    WorkflowCancelled(WorkflowTerminalOutcome),
192    #[error(transparent)]
193    WorkflowTerminated(WorkflowTerminalOutcome),
194    #[error(transparent)]
195    WorkflowTimedOut(WorkflowTerminalOutcome),
196    #[error(transparent)]
197    ActivityTaskRejected(ActivityTaskRejection),
198    #[error("workflow handler {0:?} is not registered")]
199    WorkflowNotRegistered(String),
200    #[error("activity handler {0:?} is not registered")]
201    ActivityNotRegistered(String),
202    #[error(
203        "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
204    )]
205    HandlerType {
206        handler_kind: HandlerKind,
207        handler_name: String,
208        value_kind: HandlerValueKind,
209        rust_type: &'static str,
210        message: String,
211    },
212    #[error("workflow future yielded without emitting a durable command")]
213    WorkflowYieldedWithoutCommand,
214    #[error(
215        "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
216    )]
217    MissingWorkflowCommandIdentity,
218    #[error("workflow state lock is poisoned")]
219    WorkflowStatePoisoned,
220    #[error("timer duration is too large for the worker protocol")]
221    TimerDurationOverflow,
222    #[error(transparent)]
223    InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
224    #[error(transparent)]
225    InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
226    #[error("operation timed out")]
227    Timeout,
228    #[error(
229        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
230    )]
231    MissingRoleCredentials {
232        role: &'static str,
233        opposite_role: &'static str,
234    },
235    #[error("worker loop error: {0}")]
236    WorkerLoop(String),
237    #[error(
238        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
239    )]
240    UnsupportedUpdateValidators { workflow_type: String },
241    #[error("{primary}; worker deregistration also failed: {deregistration}")]
242    WorkerShutdown {
243        primary: Box<Error>,
244        deregistration: Box<Error>,
245    },
246    #[error("invalid child workflow options: {0}")]
247    InvalidChildWorkflowOptions(String),
248    #[error("invalid workflow memo update: {0}")]
249    InvalidMemoUpdate(String),
250    #[error(
251        "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
252    )]
253    WorkflowMemoUpdatesUnavailable,
254    #[error(transparent)]
255    InvalidActivityOptions(ActivityOptionsError),
256    #[error(transparent)]
257    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
258    #[doc(hidden)]
259    #[error("workflow requested continue as new")]
260    ContinueAsNew(ContinueAsNewRequest),
261}
262
263/// Validation failure for a durable condition-wait definition.
264#[derive(Clone, Debug, Error, PartialEq, Eq)]
265pub enum ConditionWaitOptionsError {
266    #[error("condition_key must be non-empty")]
267    EmptyKey,
268    #[error("condition_definition_fingerprint must be non-empty")]
269    EmptyPredicateIdentity,
270    #[error("condition timeout is too large for the worker protocol")]
271    TimeoutOverflow,
272}
273
274/// Stable identity and optional durable timeout for a condition wait.
275///
276/// `predicate_identity` is recorded as the worker protocol's
277/// `condition_definition_fingerprint` and must change whenever predicate
278/// behavior changes. Prefer the [`wait_condition!`] macro when the predicate
279/// is written inline; it derives this identity from the predicate tokens.
280#[derive(Clone, Debug, PartialEq, Eq)]
281pub struct ConditionWaitOptions {
282    condition_key: String,
283    predicate_identity: String,
284    timeout: Option<Duration>,
285}
286
287impl ConditionWaitOptions {
288    pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
289        Self {
290            condition_key: condition_key.into(),
291            predicate_identity: predicate_identity.into(),
292            timeout: None,
293        }
294    }
295
296    pub fn timeout(mut self, timeout: Duration) -> Self {
297        self.timeout = Some(timeout);
298        self
299    }
300
301    fn validate(
302        &self,
303    ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
304        let condition_key = self.condition_key.trim();
305        if condition_key.is_empty() {
306            return Err(ConditionWaitOptionsError::EmptyKey);
307        }
308        let predicate_identity = self.predicate_identity.trim();
309        if predicate_identity.is_empty() {
310            return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
311        }
312        let timeout_seconds = self
313            .timeout
314            .map(|timeout| {
315                timeout
316                    .as_secs()
317                    .checked_add(u64::from(timeout.subsec_nanos() > 0))
318                    .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
319            })
320            .transpose()?;
321
322        Ok(ValidatedConditionWaitOptions {
323            condition_key: condition_key.to_string(),
324            predicate_identity: predicate_identity.to_string(),
325            timeout_seconds,
326        })
327    }
328}
329
330#[derive(Clone, Debug, PartialEq, Eq)]
331struct ValidatedConditionWaitOptions {
332    condition_key: String,
333    predicate_identity: String,
334    timeout_seconds: Option<u64>,
335}
336
337const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
338
339/// Unambiguous terminal result of a durable condition wait.
340#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
341#[serde(rename_all = "snake_case")]
342pub enum ConditionWaitResult {
343    Satisfied,
344    TimedOut,
345}
346
347impl ConditionWaitResult {
348    pub fn is_satisfied(self) -> bool {
349        self == Self::Satisfied
350    }
351
352    pub fn is_timed_out(self) -> bool {
353        self == Self::TimedOut
354    }
355}
356
357/// Build the stable condition definition identity used by [`wait_condition!`].
358#[doc(hidden)]
359pub fn __condition_definition_fingerprint(source: &str) -> String {
360    let mut digest = Sha256::new();
361    digest.update(b"durable-workflow-rust.wait-condition.v1\0");
362    digest.update(source.as_bytes());
363    format!("sha256:{:x}", digest.finalize())
364}
365
366/// Create a durable condition wait whose predicate definition is fingerprinted
367/// from its inline Rust tokens.
368///
369/// The returned [`ConditionWaitCall`] must be awaited. The timeout form is
370/// `wait_condition!(ctx, "approval", timeout: duration, || predicate)`.
371#[macro_export]
372macro_rules! wait_condition {
373    ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
374        $ctx.wait_condition(
375            $crate::ConditionWaitOptions::new(
376                $key,
377                $crate::__condition_definition_fingerprint(concat!(
378                    module_path!(),
379                    "\0",
380                    stringify!($predicate)
381                )),
382            )
383            .timeout($timeout),
384            $predicate,
385        )
386    }};
387    ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
388        $ctx.wait_condition(
389            $crate::ConditionWaitOptions::new(
390                $key,
391                $crate::__condition_definition_fingerprint(concat!(
392                    module_path!(),
393                    "\0",
394                    stringify!($predicate)
395                )),
396            ),
397            $predicate,
398        )
399    }};
400}
401
402const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
403const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
404const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
405const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
406const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
407
408/// Validation failure for a typed workflow search-attribute update.
409#[derive(Clone, Debug, Error, PartialEq, Eq)]
410pub enum SearchAttributeUpdateError {
411    #[error("search-attribute update requires at least one attribute")]
412    Empty,
413    #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
414    InvalidKey(String),
415    #[error("search-attribute update exceeds the limit of 100 attributes")]
416    TooManyAttributes,
417    #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
418    ValueTooLong {
419        key: String,
420        kind: &'static str,
421        limit: usize,
422    },
423    #[error(
424        "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
425    )]
426    EmptyString(String),
427    #[error("search attribute {0:?} has a non-finite float value")]
428    NonFiniteFloat(String),
429    #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
430    InvalidDateTime(String),
431    #[error("search-attribute update exceeds the 65536-byte protocol limit")]
432    PayloadTooLarge,
433}
434
435/// One public typed search-attribute value.
436#[derive(Clone, Debug, PartialEq)]
437pub enum SearchAttributeValue {
438    String(String),
439    Keyword(String),
440    KeywordList(Vec<String>),
441    Int(i64),
442    Float(f64),
443    Bool(bool),
444    DateTime(String),
445    Delete,
446}
447
448impl SearchAttributeValue {
449    fn type_name(&self) -> Option<&'static str> {
450        match self {
451            Self::String(_) => Some("string"),
452            Self::Keyword(_) => Some("keyword"),
453            Self::KeywordList(_) => Some("keyword_list"),
454            Self::Int(_) => Some("int"),
455            Self::Float(_) => Some("float"),
456            Self::Bool(_) => Some("bool"),
457            Self::DateTime(_) => Some("datetime"),
458            Self::Delete => None,
459        }
460    }
461
462    fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
463        let normalize_string = |value: String, kind: &'static str, limit: usize| {
464            let value = value.trim().to_string();
465            if value.is_empty() {
466                return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
467            }
468            if value.len() > limit {
469                return Err(SearchAttributeUpdateError::ValueTooLong {
470                    key: key.to_string(),
471                    kind,
472                    limit,
473                });
474            }
475            Ok(value)
476        };
477
478        match self {
479            Self::String(value) => Ok(Self::String(normalize_string(
480                value,
481                "string",
482                MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
483            )?)),
484            Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
485                value,
486                "keyword",
487                MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
488            )?)),
489            Self::KeywordList(values) => {
490                let values = values
491                    .into_iter()
492                    .map(|value| {
493                        let value = value.trim().to_string();
494                        if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
495                            return Err(SearchAttributeUpdateError::ValueTooLong {
496                                key: key.to_string(),
497                                kind: "keyword-list entry",
498                                limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
499                            });
500                        }
501                        Ok(value)
502                    })
503                    .collect::<std::result::Result<Vec<_>, _>>()?;
504                Ok(Self::KeywordList(values))
505            }
506            Self::Float(value) if !value.is_finite() => {
507                Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
508            }
509            Self::DateTime(value) => {
510                let value =
511                    normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
512                if DateTime::parse_from_rfc3339(&value).is_err() {
513                    return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
514                }
515                Ok(Self::DateTime(value))
516            }
517            value => Ok(value),
518        }
519    }
520
521    fn into_json(self) -> Value {
522        match self {
523            Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
524                Value::String(value)
525            }
526            Self::KeywordList(values) => {
527                Value::Array(values.into_iter().map(Value::String).collect())
528            }
529            Self::Int(value) => json!(value),
530            Self::Float(value) => json!(value),
531            Self::Bool(value) => json!(value),
532            Self::Delete => Value::Null,
533        }
534    }
535}
536
537/// Validated typed workflow-side search-attribute mutation.
538#[derive(Clone, Debug, Default, PartialEq)]
539pub struct SearchAttributeUpdate {
540    attributes: BTreeMap<String, SearchAttributeValue>,
541}
542
543impl SearchAttributeUpdate {
544    pub fn new() -> Self {
545        Self::default()
546    }
547
548    pub fn set(
549        mut self,
550        key: impl Into<String>,
551        value: SearchAttributeValue,
552    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
553        let key = key.into();
554        validate_search_attribute_key(&key)?;
555        if !self.attributes.contains_key(&key)
556            && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
557        {
558            return Err(SearchAttributeUpdateError::TooManyAttributes);
559        }
560        self.attributes.insert(key.clone(), value.normalized(&key)?);
561        self.validate_size()?;
562        Ok(self)
563    }
564
565    pub fn string(
566        self,
567        key: impl Into<String>,
568        value: impl Into<String>,
569    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
570        self.set(key, SearchAttributeValue::String(value.into()))
571    }
572
573    pub fn keyword(
574        self,
575        key: impl Into<String>,
576        value: impl Into<String>,
577    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
578        self.set(key, SearchAttributeValue::Keyword(value.into()))
579    }
580
581    pub fn keyword_list<I, V>(
582        self,
583        key: impl Into<String>,
584        values: I,
585    ) -> std::result::Result<Self, SearchAttributeUpdateError>
586    where
587        I: IntoIterator<Item = V>,
588        V: Into<String>,
589    {
590        self.set(
591            key,
592            SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
593        )
594    }
595
596    pub fn int(
597        self,
598        key: impl Into<String>,
599        value: i64,
600    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
601        self.set(key, SearchAttributeValue::Int(value))
602    }
603
604    pub fn float(
605        self,
606        key: impl Into<String>,
607        value: f64,
608    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
609        self.set(key, SearchAttributeValue::Float(value))
610    }
611
612    pub fn bool(
613        self,
614        key: impl Into<String>,
615        value: bool,
616    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
617        self.set(key, SearchAttributeValue::Bool(value))
618    }
619
620    pub fn datetime(
621        self,
622        key: impl Into<String>,
623        value: impl Into<String>,
624    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
625        self.set(key, SearchAttributeValue::DateTime(value.into()))
626    }
627
628    pub fn delete(
629        self,
630        key: impl Into<String>,
631    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
632        self.set(key, SearchAttributeValue::Delete)
633    }
634
635    fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
636        let (attributes, _) = self.clone().into_wire_parts();
637        if serde_json::to_vec(&attributes)
638            .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
639            .unwrap_or(true)
640        {
641            return Err(SearchAttributeUpdateError::PayloadTooLarge);
642        }
643        Ok(())
644    }
645
646    fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
647        let mut attributes = serde_json::Map::new();
648        let mut attribute_types = BTreeMap::new();
649        for (key, value) in self.attributes {
650            if let Some(type_name) = value.type_name() {
651                attribute_types.insert(key.clone(), type_name.to_string());
652            }
653            attributes.insert(key, value.into_json());
654        }
655        (Value::Object(attributes), attribute_types)
656    }
657
658    fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
659        if self.attributes.is_empty() {
660            return Err(SearchAttributeUpdateError::Empty);
661        }
662        self.validate_size()
663    }
664}
665
666fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
667    let valid = !key.is_empty()
668        && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
669        && key
670            .bytes()
671            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
672    if valid {
673        Ok(())
674    } else {
675        Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
676    }
677}
678
679/// The registered handler family reported by [`Error::HandlerType`].
680#[derive(Clone, Copy, Debug, PartialEq, Eq)]
681pub enum HandlerKind {
682    Workflow,
683    Activity,
684}
685
686impl std::fmt::Display for HandlerKind {
687    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
688        formatter.write_str(match self {
689            Self::Workflow => "workflow",
690            Self::Activity => "activity",
691        })
692    }
693}
694
695/// Whether a typed handler failed to adapt its input or result.
696#[derive(Clone, Copy, Debug, PartialEq, Eq)]
697pub enum HandlerValueKind {
698    Input,
699    Result,
700}
701
702impl std::fmt::Display for HandlerValueKind {
703    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
704        formatter.write_str(match self {
705            Self::Input => "input",
706            Self::Result => "result",
707        })
708    }
709}
710
711/// The lifecycle command sent to a workflow execution.
712#[derive(Clone, Copy, Debug, PartialEq, Eq)]
713pub enum WorkflowCommandKind {
714    Cancel,
715    Terminate,
716}
717
718impl WorkflowCommandKind {
719    fn as_str(self) -> &'static str {
720        match self {
721            Self::Cancel => "cancel",
722            Self::Terminate => "terminate",
723        }
724    }
725}
726
727/// Optional structured fields for a cancellation or termination request.
728#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
729pub struct WorkflowCommandOptions {
730    #[serde(skip_serializing_if = "Option::is_none")]
731    pub reason: Option<String>,
732    #[serde(skip_serializing_if = "Option::is_none")]
733    pub request_id: Option<String>,
734}
735
736/// Server-enforced timeout policy for a workflow start.
737///
738/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
739/// only bounds how long the caller waits. A server deadline produces a terminal
740/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
741/// `run_timeout`.
742#[derive(Clone, Debug, PartialEq, Eq)]
743pub struct WorkflowStartOptions {
744    pub execution_timeout_seconds: u64,
745    pub run_timeout_seconds: u64,
746}
747
748impl Default for WorkflowStartOptions {
749    fn default() -> Self {
750        Self {
751            execution_timeout_seconds: 3600,
752            run_timeout_seconds: 600,
753        }
754    }
755}
756
757impl WorkflowStartOptions {
758    pub fn new() -> Self {
759        Self::default()
760    }
761
762    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
763        self.execution_timeout_seconds = seconds;
764        self
765    }
766
767    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
768        self.run_timeout_seconds = seconds;
769        self
770    }
771
772    fn validate(&self) -> Result<()> {
773        if self.execution_timeout_seconds == 0 {
774            return Err(Error::Codec(
775                "execution_timeout_seconds must be at least 1".to_string(),
776            ));
777        }
778        if self.run_timeout_seconds == 0 {
779            return Err(Error::Codec(
780                "run_timeout_seconds must be at least 1".to_string(),
781            ));
782        }
783        if self.run_timeout_seconds > self.execution_timeout_seconds {
784            return Err(Error::Codec(
785                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
786            ));
787        }
788
789        Ok(())
790    }
791}
792
793/// Optional routing overrides for a continue-as-new transition.
794///
795/// Omitted values retain the current workflow type and task queue. Server-owned
796/// instance metadata is not accepted here and is carried by the server.
797#[derive(Clone, Debug, Default, PartialEq, Eq)]
798pub struct ContinueAsNewOptions {
799    pub workflow_type: Option<String>,
800    pub task_queue: Option<String>,
801}
802
803impl ContinueAsNewOptions {
804    pub fn new() -> Self {
805        Self::default()
806    }
807
808    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
809        self.workflow_type = Some(workflow_type.into());
810        self
811    }
812
813    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
814        self.task_queue = Some(task_queue.into());
815        self
816    }
817
818    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
819        for (field, value) in [
820            ("workflow_type", self.workflow_type.as_deref()),
821            ("task_queue", self.task_queue.as_deref()),
822        ] {
823            if value.is_some_and(|value| value.trim().is_empty()) {
824                return Err(ContinueAsNewOptionsError {
825                    field,
826                    message: format!("{field} must not be empty"),
827                });
828            }
829        }
830        Ok(())
831    }
832}
833
834/// A stable validation error raised before a continue-as-new command is emitted.
835#[derive(Clone, Debug, Error, PartialEq, Eq)]
836#[error("invalid continue-as-new option {field}: {message}")]
837pub struct ContinueAsNewOptionsError {
838    pub field: &'static str,
839    pub message: String,
840}
841
842/// Public history-budget information attached to the current workflow task.
843#[derive(Clone, Debug, Default, PartialEq, Eq)]
844pub struct WorkflowHistoryBudget {
845    pub event_count: u64,
846    pub size_bytes: Option<u64>,
847    pub continue_as_new_recommended: bool,
848    pub pressure: Option<String>,
849}
850
851#[doc(hidden)]
852#[derive(Clone, Debug)]
853pub struct ContinueAsNewRequest {
854    arguments: AvroValue,
855    options: ContinueAsNewOptions,
856}
857
858impl WorkflowCommandOptions {
859    pub fn new() -> Self {
860        Self::default()
861    }
862
863    pub fn reason(mut self, reason: impl Into<String>) -> Self {
864        self.reason = Some(reason.into());
865        self
866    }
867
868    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
869        self.request_id = Some(request_id.into());
870        self
871    }
872}
873
874/// The accepted, machine-readable result of a lifecycle command.
875#[derive(Clone, Debug, PartialEq)]
876pub struct WorkflowCommandResult {
877    pub command: WorkflowCommandKind,
878    pub workflow_id: String,
879    pub run_id: Option<String>,
880    pub outcome: Option<String>,
881    pub reason: Option<String>,
882    pub command_status: Option<String>,
883    pub raw: Value,
884}
885
886/// A new run continuing a failed run from its recorded activity boundary.
887#[derive(Clone, Debug, PartialEq)]
888pub struct WorkflowRedriveResult {
889    pub workflow_id: String,
890    pub source_run_id: String,
891    pub run_id: String,
892    pub resume_step_sequence: u64,
893    pub raw: Value,
894}
895
896/// A stable rejection returned by instance- or selected-run lifecycle commands.
897#[derive(Clone, Debug, Error)]
898#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
899pub struct WorkflowCommandRejection {
900    pub command: WorkflowCommandKind,
901    pub status: u16,
902    pub reason: String,
903    pub message: String,
904    pub workflow_id: String,
905    pub run_id: Option<String>,
906    pub target_scope: Option<String>,
907    pub body: Value,
908}
909
910/// Stable terminal categories returned by [`WorkflowHandle::result`].
911#[derive(Clone, Copy, Debug, PartialEq, Eq)]
912pub enum WorkflowTerminalKind {
913    Failed,
914    Cancelled,
915    Terminated,
916    TimedOut,
917}
918
919/// A typed terminal workflow outcome with durable identity and failure metadata.
920///
921/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
922/// of parsing its display representation. Fields remain `None` when an older
923/// server did not publish that metadata.
924#[derive(Clone, Debug, Error)]
925#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
926pub struct WorkflowTerminalOutcome {
927    pub kind: WorkflowTerminalKind,
928    pub workflow_id: String,
929    pub run_id: Option<String>,
930    pub reason: String,
931    pub failure_category: Option<String>,
932    pub failure_id: Option<String>,
933    pub exception_type: Option<String>,
934    pub exception_class: Option<String>,
935    pub non_retryable: Option<bool>,
936    pub message: Option<String>,
937    pub exception: Option<Value>,
938    pub raw: Value,
939}
940
941/// A worker-side activity settlement or heartbeat rejected by durable state.
942#[derive(Clone, Debug, Error)]
943#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
944pub struct ActivityTaskRejection {
945    pub operation: String,
946    pub status: u16,
947    pub reason: String,
948    pub task_id: String,
949    pub activity_attempt_id: String,
950    pub cancel_requested: bool,
951    pub can_continue: Option<bool>,
952    pub run_closed_reason: Option<String>,
953    pub body: Value,
954}
955
956/// Stable validation categories for [`ActivityOptions`].
957#[derive(Clone, Copy, Debug, PartialEq, Eq)]
958pub enum ActivityOptionsErrorKind {
959    EmptyTaskQueue,
960    EmptyRetryPolicy,
961    InvalidMaxAttempts,
962    BackoffWithoutRetryBudget,
963    TooManyBackoffIntervals,
964    InvalidBackoffCoefficient,
965    BackoffGenerationTooLarge,
966    BackoffOverflow,
967    EmptyNonRetryableErrorType,
968    TimeoutNotPositive,
969    TimeoutOverflow,
970    TimeoutOrder,
971}
972
973/// A machine-readable activity-options validation failure.
974#[derive(Clone, Debug, Error, PartialEq, Eq)]
975#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
976pub struct ActivityOptionsError {
977    pub kind: ActivityOptionsErrorKind,
978    pub field: Option<&'static str>,
979    pub message: String,
980}
981
982impl ActivityOptionsError {
983    fn new(
984        kind: ActivityOptionsErrorKind,
985        field: Option<&'static str>,
986        message: impl Into<String>,
987    ) -> Self {
988        Self {
989            kind,
990            field,
991            message: message.into(),
992        }
993    }
994}
995
996/// Stable terminal categories returned when an awaited activity does not succeed.
997#[derive(Clone, Copy, Debug, PartialEq, Eq)]
998pub enum ActivityFailureKind {
999    Failed,
1000    Cancelled,
1001    TimedOut,
1002}
1003
1004/// A stable, machine-readable terminal activity failure.
1005///
1006/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
1007/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
1008#[derive(Clone, Debug, Error, PartialEq)]
1009#[error("activity failed ({reason}): {message}")]
1010pub struct ActivityFailure {
1011    pub kind: ActivityFailureKind,
1012    pub reason: String,
1013    pub message: String,
1014    pub activity_execution_id: Option<String>,
1015    pub activity_attempt_id: Option<String>,
1016    pub activity_type: Option<String>,
1017    pub activity_class: Option<String>,
1018    pub attempt_number: Option<u64>,
1019    pub failure_id: Option<String>,
1020    pub failure_category: Option<String>,
1021    pub timeout_kind: Option<String>,
1022    pub non_retryable: bool,
1023    pub exception_type: Option<String>,
1024    pub exception_class: Option<String>,
1025    pub code: Option<Value>,
1026    pub exception: Option<Value>,
1027}
1028
1029/// Stable terminal categories returned when an awaited child does not succeed.
1030#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1031pub enum ChildWorkflowFailureKind {
1032    Failed,
1033    Cancelled,
1034    Terminated,
1035}
1036
1037/// A stable, machine-readable child workflow failure delivered to its parent.
1038///
1039/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
1040/// of parsing the display message. Child and parent identifiers retain the
1041/// relationship recorded in durable history across worker restarts.
1042#[derive(Clone, Debug, Error)]
1043#[error("child workflow failed ({reason}): {message}")]
1044pub struct ChildWorkflowFailure {
1045    pub kind: ChildWorkflowFailureKind,
1046    pub reason: String,
1047    pub message: String,
1048    pub parent_workflow_id: Option<String>,
1049    pub parent_workflow_run_id: Option<String>,
1050    pub child_workflow_id: Option<String>,
1051    pub child_workflow_run_id: Option<String>,
1052    pub child_workflow_type: Option<String>,
1053    pub failure_id: Option<String>,
1054    pub failure_category: Option<String>,
1055    pub exception_type: Option<String>,
1056    pub exception_class: Option<String>,
1057    pub non_retryable: bool,
1058    pub code: Option<Value>,
1059    pub exception: Option<Value>,
1060}
1061
1062/// The identity of one durable workflow execution.
1063#[derive(Clone, Debug, PartialEq, Eq)]
1064pub struct WorkflowIdentity {
1065    pub workflow_id: Option<String>,
1066    pub run_id: Option<String>,
1067}
1068
1069/// A successful child result together with its durable parent-child identity.
1070#[derive(Clone, Debug, PartialEq)]
1071pub struct ChildWorkflowResult {
1072    pub parent: WorkflowIdentity,
1073    pub child: WorkflowIdentity,
1074    pub child_workflow_type: Option<String>,
1075    pub result: Value,
1076}
1077
1078/// Lossless successful child result for fixed Avro Value workflows.
1079#[derive(Clone, Debug, PartialEq)]
1080pub struct ChildWorkflowAvroResult {
1081    pub parent: WorkflowIdentity,
1082    pub child: WorkflowIdentity,
1083    pub child_workflow_type: Option<String>,
1084    pub result: AvroValue,
1085}
1086
1087/// Stable user-facing identity for one member of a durable selection group.
1088#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1089#[serde(untagged)]
1090pub enum SelectionKey {
1091    Index(usize),
1092    Name(String),
1093}
1094
1095impl From<usize> for SelectionKey {
1096    fn from(value: usize) -> Self {
1097        Self::Index(value)
1098    }
1099}
1100
1101impl From<String> for SelectionKey {
1102    fn from(value: String) -> Self {
1103        Self::Name(value)
1104    }
1105}
1106
1107impl From<&str> for SelectionKey {
1108    fn from(value: &str) -> Self {
1109        Self::Name(value.to_string())
1110    }
1111}
1112
1113/// Typed result of explicitly awaiting a cancelled non-winning operation.
1114#[derive(Clone, Debug, Error, PartialEq, Eq)]
1115#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1116pub struct DurableOperationCancelled {
1117    pub selection_group_id: String,
1118    pub member_key: SelectionKey,
1119    pub member_index: usize,
1120    pub operation_kind: String,
1121    pub operation_identity: String,
1122}
1123
1124/// Stable identity for one enclosing deterministic parallel group.
1125///
1126/// The same fields are attached to every ordinary activity, timer, or child
1127/// workflow command in the group. Nested leaves carry an outer-to-inner path;
1128/// no Rust-specific wire command is introduced.
1129#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1130pub struct ParallelGroupMetadata {
1131    pub parallel_group_id: String,
1132    pub parallel_group_kind: String,
1133    pub parallel_group_base_sequence: u64,
1134    pub parallel_group_size: usize,
1135    pub parallel_group_index: usize,
1136    #[serde(default, skip_serializing_if = "Option::is_none")]
1137    pub parallel_group_mode: Option<String>,
1138    #[serde(default, skip_serializing_if = "Option::is_none")]
1139    pub selection_member_key: Option<SelectionKey>,
1140    #[serde(default, skip_serializing_if = "Option::is_none")]
1141    pub selection_member_index: Option<usize>,
1142    #[serde(default, skip_serializing_if = "Option::is_none")]
1143    pub selection_member_base_sequence: Option<u64>,
1144    #[serde(default, skip_serializing_if = "Option::is_none")]
1145    pub selection_member_size: Option<usize>,
1146    #[serde(default, skip_serializing_if = "Option::is_none")]
1147    pub selection_member_kind: Option<String>,
1148}
1149
1150/// One input-ordered result returned by [`WorkflowContext::parallel`].
1151#[derive(Clone, Debug, PartialEq)]
1152pub enum ParallelResult {
1153    Activity(Value),
1154    ChildWorkflow(ChildWorkflowResult),
1155    Timer,
1156    Signal(Vec<Value>),
1157    Condition(ConditionWaitResult),
1158    Group(Vec<ParallelResult>),
1159}
1160
1161/// Lossless fixed-Avro counterpart to [`ParallelResult`].
1162#[derive(Clone, Debug, PartialEq)]
1163pub enum ParallelAvroResult {
1164    Activity(AvroValue),
1165    ChildWorkflow(ChildWorkflowAvroResult),
1166    Timer,
1167    Signal(Vec<AvroValue>),
1168    Condition(ConditionWaitResult),
1169    Group(Vec<ParallelAvroResult>),
1170}
1171
1172impl ParallelAvroResult {
1173    fn into_json_result(self) -> Result<ParallelResult> {
1174        match self {
1175            Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1176            Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1177                parent: result.parent,
1178                child: result.child,
1179                child_workflow_type: result.child_workflow_type,
1180                result: result.result.into_json()?,
1181            })),
1182            Self::Timer => Ok(ParallelResult::Timer),
1183            Self::Signal(values) => Ok(ParallelResult::Signal(
1184                values
1185                    .into_iter()
1186                    .map(AvroValue::into_json)
1187                    .collect::<Result<Vec<_>>>()?,
1188            )),
1189            Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1190            Self::Group(results) => Ok(ParallelResult::Group(
1191                results
1192                    .into_iter()
1193                    .map(Self::into_json_result)
1194                    .collect::<Result<Vec<_>>>()?,
1195            )),
1196        }
1197    }
1198}
1199
1200/// One successful leaf retained when another parallel member failed.
1201#[derive(Clone, Debug, PartialEq)]
1202pub struct ParallelCompletion {
1203    pub member_path: Vec<usize>,
1204    pub result: ParallelResult,
1205}
1206
1207/// A deterministic join failed after some siblings had already completed.
1208///
1209/// `cause` retains the typed activity, child-workflow, cancellation, or codec
1210/// error. `completed` is declaration ordered and contains only durable
1211/// successes observed in the same replay. Late sibling completions can add
1212/// entries on a later replay without changing `member_path` or the selected
1213/// positional failure.
1214#[derive(Debug, Error)]
1215#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1216pub struct ParallelFailure {
1217    pub group_id: String,
1218    pub member_path: Vec<usize>,
1219    pub group_path: Vec<ParallelGroupMetadata>,
1220    pub completed: Vec<ParallelCompletion>,
1221    #[source]
1222    pub cause: Box<Error>,
1223}
1224
1225/// Stable validation error returned before an invalid group emits commands.
1226#[derive(Clone, Debug, Error, PartialEq, Eq)]
1227#[error("invalid deterministic parallel group ({reason}): {message}")]
1228pub struct ParallelGroupError {
1229    pub reason: &'static str,
1230    pub member_path: Vec<usize>,
1231    pub message: String,
1232}
1233
1234/// Cooperative workflow cancellation observed at an author-controlled point.
1235#[derive(Clone, Debug, Error, PartialEq, Eq)]
1236#[error("workflow cancellation was requested")]
1237pub struct WorkflowCancellationRequested;
1238
1239/// A forward saga failure followed by a terminal compensation failure.
1240#[derive(Debug, Error)]
1241#[error(
1242    "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1243)]
1244pub struct SagaCompensationFailure {
1245    pub initiating_failure: Box<Error>,
1246    pub compensation_failure: Box<Error>,
1247    pub compensation_activity_type: String,
1248    pub compensation_registration_order: usize,
1249}
1250
1251/// Server behavior when a parent closes while its child is still open.
1252#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1253pub enum ParentClosePolicy {
1254    #[default]
1255    Abandon,
1256    RequestCancel,
1257    Terminate,
1258}
1259
1260impl ParentClosePolicy {
1261    fn as_str(self) -> &'static str {
1262        match self {
1263            Self::Abandon => "abandon",
1264            Self::RequestCancel => "request_cancel",
1265            Self::Terminate => "terminate",
1266        }
1267    }
1268}
1269
1270/// Durable retry policy for one child workflow invocation.
1271#[derive(Clone, Debug, Default, PartialEq, Eq)]
1272pub struct ChildWorkflowRetryPolicy {
1273    pub max_attempts: Option<u32>,
1274    pub backoff_seconds: Vec<u64>,
1275    pub non_retryable_error_types: Vec<String>,
1276}
1277
1278/// Options recorded with a child-workflow command.
1279///
1280/// The task queue is mandatory so routing is explicit and replay-stable.
1281#[derive(Clone, Debug, PartialEq, Eq)]
1282pub struct ChildWorkflowOptions {
1283    pub task_queue: String,
1284    pub parent_close_policy: ParentClosePolicy,
1285    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1286    pub execution_timeout_seconds: Option<u64>,
1287    pub run_timeout_seconds: Option<u64>,
1288}
1289
1290impl ChildWorkflowOptions {
1291    pub fn new(task_queue: impl Into<String>) -> Self {
1292        Self {
1293            task_queue: task_queue.into(),
1294            parent_close_policy: ParentClosePolicy::Abandon,
1295            retry_policy: None,
1296            execution_timeout_seconds: None,
1297            run_timeout_seconds: None,
1298        }
1299    }
1300
1301    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1302        self.parent_close_policy = policy;
1303        self
1304    }
1305
1306    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1307        self.retry_policy = Some(policy);
1308        self
1309    }
1310
1311    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1312        self.execution_timeout_seconds = Some(seconds);
1313        self
1314    }
1315
1316    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1317        self.run_timeout_seconds = Some(seconds);
1318        self
1319    }
1320}
1321
1322/// Backoff intervals for one durable activity retry policy.
1323#[derive(Clone, Debug, PartialEq, Eq)]
1324pub enum ActivityBackoff {
1325    /// Use these intervals between attempts. The server repeats the final
1326    /// interval if the retry budget contains more attempts than entries.
1327    Explicit(Vec<Duration>),
1328    /// Generate one interval for every retry using integer exponential growth.
1329    Exponential {
1330        initial_interval: Duration,
1331        coefficient: u32,
1332        maximum_interval: Option<Duration>,
1333    },
1334}
1335
1336/// Durable server-side retry policy for one activity execution.
1337#[derive(Clone, Debug, Default, PartialEq, Eq)]
1338pub struct ActivityRetryPolicy {
1339    pub max_attempts: Option<u32>,
1340    pub backoff: Option<ActivityBackoff>,
1341    pub non_retryable_error_types: Vec<String>,
1342}
1343
1344impl ActivityRetryPolicy {
1345    /// Start a policy with a finite attempt budget, including the first attempt.
1346    pub fn new(max_attempts: u32) -> Self {
1347        Self {
1348            max_attempts: Some(max_attempts),
1349            ..Self::default()
1350        }
1351    }
1352
1353    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1354        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1355        self
1356    }
1357
1358    pub fn exponential_backoff(
1359        mut self,
1360        initial_interval: Duration,
1361        coefficient: u32,
1362        maximum_interval: Option<Duration>,
1363    ) -> Self {
1364        self.backoff = Some(ActivityBackoff::Exponential {
1365            initial_interval,
1366            coefficient,
1367            maximum_interval,
1368        });
1369        self
1370    }
1371
1372    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1373        self.non_retryable_error_types.push(error_type.into());
1374        self
1375    }
1376
1377    pub fn non_retryable_error_types(
1378        mut self,
1379        error_types: impl IntoIterator<Item = impl Into<String>>,
1380    ) -> Self {
1381        self.non_retryable_error_types
1382            .extend(error_types.into_iter().map(Into::into));
1383        self
1384    }
1385}
1386
1387/// Options recorded atomically on one deterministic `schedule_activity` command.
1388///
1389/// Durations are rounded up to whole seconds when encoded, so the server never
1390/// receives a shorter timeout or backoff than the caller requested.
1391#[derive(Clone, Debug, Default, PartialEq, Eq)]
1392pub struct ActivityOptions {
1393    pub task_queue: Option<String>,
1394    pub retry_policy: Option<ActivityRetryPolicy>,
1395    pub start_to_close_timeout: Option<Duration>,
1396    pub schedule_to_start_timeout: Option<Duration>,
1397    pub schedule_to_close_timeout: Option<Duration>,
1398    pub heartbeat_timeout: Option<Duration>,
1399}
1400
1401impl ActivityOptions {
1402    pub fn new() -> Self {
1403        Self::default()
1404    }
1405
1406    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1407        self.task_queue = Some(task_queue.into());
1408        self
1409    }
1410
1411    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1412        self.retry_policy = Some(policy);
1413        self
1414    }
1415
1416    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1417        self.start_to_close_timeout = Some(timeout);
1418        self
1419    }
1420
1421    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1422        self.schedule_to_start_timeout = Some(timeout);
1423        self
1424    }
1425
1426    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1427        self.schedule_to_close_timeout = Some(timeout);
1428        self
1429    }
1430
1431    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1432        self.heartbeat_timeout = Some(timeout);
1433        self
1434    }
1435
1436    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1437        if self
1438            .task_queue
1439            .as_deref()
1440            .is_some_and(|queue| queue.trim().is_empty())
1441        {
1442            return Err(ActivityOptionsError::new(
1443                ActivityOptionsErrorKind::EmptyTaskQueue,
1444                Some("task_queue"),
1445                "task_queue must not be empty",
1446            ));
1447        }
1448
1449        for (field, value) in [
1450            ("start_to_close_timeout", self.start_to_close_timeout),
1451            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1452            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1453            ("heartbeat_timeout", self.heartbeat_timeout),
1454        ] {
1455            if value.is_some_and(|value| value.is_zero()) {
1456                return Err(ActivityOptionsError::new(
1457                    ActivityOptionsErrorKind::TimeoutNotPositive,
1458                    Some(field),
1459                    format!("{field} must be positive"),
1460                ));
1461            }
1462        }
1463
1464        validate_timeout_order(
1465            "heartbeat_timeout",
1466            self.heartbeat_timeout,
1467            "start_to_close_timeout",
1468            self.start_to_close_timeout,
1469        )?;
1470        validate_timeout_order(
1471            "start_to_close_timeout",
1472            self.start_to_close_timeout,
1473            "schedule_to_close_timeout",
1474            self.schedule_to_close_timeout,
1475        )?;
1476        validate_timeout_order(
1477            "schedule_to_start_timeout",
1478            self.schedule_to_start_timeout,
1479            "schedule_to_close_timeout",
1480            self.schedule_to_close_timeout,
1481        )?;
1482
1483        Ok(ValidatedActivityOptions {
1484            task_queue: self.task_queue.clone(),
1485            retry_policy: self
1486                .retry_policy
1487                .as_ref()
1488                .map(validate_activity_retry_policy)
1489                .transpose()?,
1490            start_to_close_timeout: timeout_seconds(
1491                "start_to_close_timeout",
1492                self.start_to_close_timeout,
1493            )?,
1494            schedule_to_start_timeout: timeout_seconds(
1495                "schedule_to_start_timeout",
1496                self.schedule_to_start_timeout,
1497            )?,
1498            schedule_to_close_timeout: timeout_seconds(
1499                "schedule_to_close_timeout",
1500                self.schedule_to_close_timeout,
1501            )?,
1502            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1503        })
1504    }
1505}
1506
1507/// A deferred durable leaf or nested group for [`WorkflowContext::parallel`].
1508///
1509/// Constructors capture arguments but perform no I/O. The join validates the
1510/// complete tree, attaches the existing parallel-group metadata to every
1511/// ordinary command, schedules all leaves, and then suspends.
1512pub enum ParallelOperation {
1513    Activity {
1514        activity_type: String,
1515        options: ActivityOptions,
1516        arguments: Result<AvroValue>,
1517    },
1518    ChildWorkflow {
1519        workflow_type: String,
1520        options: ChildWorkflowOptions,
1521        arguments: Result<AvroValue>,
1522    },
1523    Timer(Duration),
1524    Signal(String),
1525    Condition {
1526        options: ConditionWaitOptions,
1527        predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1528    },
1529    Group(Vec<ParallelOperation>),
1530}
1531
1532impl ParallelOperation {
1533    pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1534        Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1535    }
1536
1537    pub fn activity_with_options<T: Serialize>(
1538        activity_type: impl Into<String>,
1539        options: ActivityOptions,
1540        args: T,
1541    ) -> Self {
1542        Self::Activity {
1543            activity_type: activity_type.into(),
1544            options,
1545            arguments: AvroValue::from_serialize(&args),
1546        }
1547    }
1548
1549    pub fn child_workflow<T: Serialize>(
1550        workflow_type: impl Into<String>,
1551        options: ChildWorkflowOptions,
1552        args: T,
1553    ) -> Self {
1554        Self::ChildWorkflow {
1555            workflow_type: workflow_type.into(),
1556            options,
1557            arguments: AvroValue::from_serialize(&args),
1558        }
1559    }
1560
1561    pub fn timer(duration: Duration) -> Self {
1562        Self::Timer(duration)
1563    }
1564
1565    pub fn signal(signal_name: impl Into<String>) -> Self {
1566        Self::Signal(signal_name.into())
1567    }
1568
1569    pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1570    where
1571        F: Fn() -> Result<bool> + Send + 'static,
1572    {
1573        Self::Condition {
1574            options,
1575            predicate: Box::new(predicate),
1576        }
1577    }
1578
1579    pub fn group(operations: Vec<ParallelOperation>) -> Self {
1580        Self::Group(operations)
1581    }
1582}
1583
1584#[derive(Clone, Debug)]
1585struct ValidatedActivityOptions {
1586    task_queue: Option<String>,
1587    retry_policy: Option<Value>,
1588    start_to_close_timeout: Option<u64>,
1589    schedule_to_start_timeout: Option<u64>,
1590    schedule_to_close_timeout: Option<u64>,
1591    heartbeat_timeout: Option<u64>,
1592}
1593
1594fn validate_timeout_order(
1595    smaller_name: &'static str,
1596    smaller: Option<Duration>,
1597    larger_name: &'static str,
1598    larger: Option<Duration>,
1599) -> std::result::Result<(), ActivityOptionsError> {
1600    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1601        return Err(ActivityOptionsError::new(
1602            ActivityOptionsErrorKind::TimeoutOrder,
1603            Some(smaller_name),
1604            format!("{smaller_name} must be <= {larger_name}"),
1605        ));
1606    }
1607    Ok(())
1608}
1609
1610fn timeout_seconds(
1611    field: &'static str,
1612    value: Option<Duration>,
1613) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1614    value
1615        .map(|value| {
1616            activity_protocol_seconds(value).ok_or_else(|| {
1617                ActivityOptionsError::new(
1618                    ActivityOptionsErrorKind::TimeoutOverflow,
1619                    Some(field),
1620                    format!("{field} is too large for the worker protocol"),
1621                )
1622            })
1623        })
1624        .transpose()
1625}
1626
1627fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1628    value
1629        .as_secs()
1630        .checked_add(u64::from(value.subsec_nanos() > 0))
1631}
1632
1633fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1634    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1635}
1636
1637fn validate_activity_retry_policy(
1638    policy: &ActivityRetryPolicy,
1639) -> std::result::Result<Value, ActivityOptionsError> {
1640    if policy.max_attempts.is_none()
1641        && policy.backoff.is_none()
1642        && policy.non_retryable_error_types.is_empty()
1643    {
1644        return Err(ActivityOptionsError::new(
1645            ActivityOptionsErrorKind::EmptyRetryPolicy,
1646            Some("retry_policy"),
1647            "retry_policy must configure at least one field",
1648        ));
1649    }
1650    if policy.max_attempts == Some(0) {
1651        return Err(ActivityOptionsError::new(
1652            ActivityOptionsErrorKind::InvalidMaxAttempts,
1653            Some("retry_policy.max_attempts"),
1654            "max_attempts must be >= 1",
1655        ));
1656    }
1657    if policy
1658        .non_retryable_error_types
1659        .iter()
1660        .any(|error_type| error_type.trim().is_empty())
1661    {
1662        return Err(ActivityOptionsError::new(
1663            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1664            Some("retry_policy.non_retryable_error_types"),
1665            "non_retryable_error_types must not contain empty values",
1666        ));
1667    }
1668
1669    let backoff_seconds = match &policy.backoff {
1670        None => None,
1671        Some(backoff) => {
1672            let max_attempts = policy.max_attempts.ok_or_else(|| {
1673                ActivityOptionsError::new(
1674                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1675                    Some("retry_policy.backoff"),
1676                    "backoff requires max_attempts",
1677                )
1678            })?;
1679            let retry_count = max_attempts.saturating_sub(1) as usize;
1680            let intervals = match backoff {
1681                ActivityBackoff::Explicit(intervals) => {
1682                    if intervals.len() > retry_count {
1683                        return Err(ActivityOptionsError::new(
1684                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
1685                            Some("retry_policy.backoff"),
1686                            "backoff interval count must not exceed max_attempts - 1",
1687                        ));
1688                    }
1689                    intervals.clone()
1690                }
1691                ActivityBackoff::Exponential {
1692                    initial_interval,
1693                    coefficient,
1694                    maximum_interval,
1695                } => {
1696                    if *coefficient < 1 {
1697                        return Err(ActivityOptionsError::new(
1698                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1699                            Some("retry_policy.backoff.coefficient"),
1700                            "backoff coefficient must be >= 1",
1701                        ));
1702                    }
1703                    if retry_count > 10_000 {
1704                        return Err(ActivityOptionsError::new(
1705                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1706                            Some("retry_policy.max_attempts"),
1707                            "generated backoff supports at most 10000 retry intervals",
1708                        ));
1709                    }
1710                    let mut current = *initial_interval;
1711                    let mut intervals = Vec::with_capacity(retry_count);
1712                    for _ in 0..retry_count {
1713                        let interval = maximum_interval
1714                            .map(|maximum| current.min(maximum))
1715                            .unwrap_or(current);
1716                        intervals.push(interval);
1717                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
1718                            break;
1719                        }
1720                        current = current.checked_mul(*coefficient).ok_or_else(|| {
1721                            ActivityOptionsError::new(
1722                                ActivityOptionsErrorKind::BackoffOverflow,
1723                                Some("retry_policy.backoff"),
1724                                "generated backoff interval overflowed",
1725                            )
1726                        })?;
1727                    }
1728                    intervals
1729                }
1730            };
1731            Some(
1732                intervals
1733                    .into_iter()
1734                    .map(|interval| {
1735                        activity_protocol_seconds(interval).ok_or_else(|| {
1736                            ActivityOptionsError::new(
1737                                ActivityOptionsErrorKind::BackoffOverflow,
1738                                Some("retry_policy.backoff"),
1739                                "backoff interval is too large for the worker protocol",
1740                            )
1741                        })
1742                    })
1743                    .collect::<std::result::Result<Vec<_>, _>>()?,
1744            )
1745        }
1746    };
1747
1748    let mut encoded = serde_json::Map::new();
1749    if let Some(max_attempts) = policy.max_attempts {
1750        encoded.insert("max_attempts".to_string(), json!(max_attempts));
1751    }
1752    if let Some(backoff_seconds) = backoff_seconds {
1753        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1754    }
1755    if !policy.non_retryable_error_types.is_empty() {
1756        let mut canonical_error_types = Vec::new();
1757        for error_type in policy
1758            .non_retryable_error_types
1759            .iter()
1760            .map(|error_type| error_type.trim())
1761        {
1762            if !canonical_error_types.contains(&error_type) {
1763                canonical_error_types.push(error_type);
1764            }
1765        }
1766        encoded.insert(
1767            "non_retryable_error_types".to_string(),
1768            json!(canonical_error_types),
1769        );
1770    }
1771    Ok(Value::Object(encoded))
1772}
1773
1774/// A stable, machine-readable failure raised when workflow code no longer
1775/// reconstructs the durable command stream recorded in history.
1776#[derive(Clone, Debug, Error)]
1777#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1778pub struct ReplayFailure {
1779    pub reason: String,
1780    pub sequence: Option<u64>,
1781    pub expected: Option<String>,
1782    pub actual: Option<String>,
1783    pub message: String,
1784}
1785
1786impl ReplayFailure {
1787    fn new(
1788        reason: impl Into<String>,
1789        sequence: Option<u64>,
1790        expected: Option<String>,
1791        actual: Option<String>,
1792        message: impl Into<String>,
1793    ) -> Self {
1794        Self {
1795            reason: reason.into(),
1796            sequence,
1797            expected,
1798            actual,
1799            message: message.into(),
1800        }
1801    }
1802}
1803
1804/// A stable, machine-readable workflow query or query-task settlement failure.
1805#[derive(Clone, Debug, Error)]
1806#[error("query failed ({reason}, HTTP {status}): {message}")]
1807pub struct QueryFailure {
1808    pub status: u16,
1809    pub reason: String,
1810    pub message: String,
1811    pub body: Value,
1812}
1813
1814/// A stable failure returned when a server rejects an SDK protocol version.
1815#[derive(Clone, Debug, Error)]
1816#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1817pub struct ProtocolFailure {
1818    pub status: u16,
1819    pub reason: String,
1820    pub message: String,
1821    pub supported_version: Option<String>,
1822    pub requested_version: Option<String>,
1823    pub body: Value,
1824}
1825
1826#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1827pub struct PayloadEnvelope {
1828    pub codec: String,
1829    pub blob: String,
1830}
1831
1832impl PayloadEnvelope {
1833    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1834        encode_payload(value, DEFAULT_CODEC)
1835    }
1836
1837    /// Encode an explicit typed value, including the bytes branch that JSON
1838    /// serialization cannot represent.
1839    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1840        encode_avro_value(value)
1841    }
1842}
1843
1844/// Native adapter for the fixed language-neutral Avro Value schema.
1845#[derive(Clone, Debug)]
1846pub enum AvroValue {
1847    Null,
1848    Boolean(bool),
1849    Long(i64),
1850    Double(f64),
1851    Bytes(Vec<u8>),
1852    String(String),
1853    Array(Vec<AvroValue>),
1854    Map(BTreeMap<String, AvroValue>),
1855}
1856
1857impl PartialEq for AvroValue {
1858    fn eq(&self, other: &Self) -> bool {
1859        match (self, other) {
1860            (Self::Null, Self::Null) => true,
1861            (Self::Boolean(left), Self::Boolean(right)) => left == right,
1862            (Self::Long(left), Self::Long(right)) => left == right,
1863            (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1864            (Self::Bytes(left), Self::Bytes(right)) => left == right,
1865            (Self::String(left), Self::String(right)) => left == right,
1866            (Self::Array(left), Self::Array(right)) => left == right,
1867            (Self::Map(left), Self::Map(right)) => left == right,
1868            _ => false,
1869        }
1870    }
1871}
1872
1873impl AvroValue {
1874    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1875        Self::from_serde_value(
1876            serde_value::to_value(value).map_err(|error| {
1877                Error::Codec(format!("could not adapt value for Avro: {error}"))
1878            })?,
1879        )
1880    }
1881
1882    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1883        use serde_value::Value as SerdeValue;
1884
1885        match value {
1886            SerdeValue::Unit => Ok(Self::Null),
1887            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1888            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1889            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1890            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1891            SerdeValue::I64(value) => Ok(Self::Long(value)),
1892            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1893            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1894            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1895            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1896                Error::Codec(
1897                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1898                        .to_string(),
1899                )
1900            }),
1901            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1902            SerdeValue::F64(value) => Self::finite_double(value),
1903            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1904            SerdeValue::String(value) => Ok(Self::String(value)),
1905            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1906            SerdeValue::Option(None) => Ok(Self::Null),
1907            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1908                Self::from_serde_value(*value)
1909            }
1910            SerdeValue::Seq(values) => values
1911                .into_iter()
1912                .map(Self::from_serde_value)
1913                .collect::<Result<Vec<_>>>()
1914                .map(Self::Array),
1915            SerdeValue::Map(values) => values
1916                .into_iter()
1917                .map(|(key, value)| {
1918                    let SerdeValue::String(key) = key else {
1919                        return Err(Error::Codec(
1920                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1921                        ));
1922                    };
1923
1924                    Ok((key, Self::from_serde_value(value)?))
1925                })
1926                .collect::<Result<BTreeMap<_, _>>>()
1927                .map(Self::Map),
1928        }
1929    }
1930
1931    fn finite_double(value: f64) -> Result<Self> {
1932        if !value.is_finite() {
1933            return Err(Error::Codec(
1934                "non_finite_float: Avro Value doubles must be finite".to_string(),
1935            ));
1936        }
1937
1938        Ok(Self::Double(value))
1939    }
1940
1941    fn into_json(self) -> Result<Value> {
1942        match self {
1943            Self::Null => Ok(Value::Null),
1944            Self::Boolean(value) => Ok(Value::Bool(value)),
1945            Self::Long(value) => Ok(Value::Number(value.into())),
1946            Self::Double(value) => serde_json::Number::from_f64(value)
1947                .map(Value::Number)
1948                .ok_or_else(|| {
1949                    Error::Codec(
1950                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1951                    )
1952                }),
1953            Self::Bytes(value) => Ok(json!({
1954                "$type": "bytes",
1955                "base64": BASE64.encode(value),
1956            })),
1957            Self::String(value) => Ok(Value::String(value)),
1958            Self::Array(values) => values
1959                .into_iter()
1960                .map(Self::into_json)
1961                .collect::<Result<Vec<_>>>()
1962                .map(Value::Array),
1963            Self::Map(values) => values
1964                .into_iter()
1965                .map(|(key, value)| Ok((key, value.into_json()?)))
1966                .collect::<Result<serde_json::Map<_, _>>>()
1967                .map(Value::Object),
1968        }
1969    }
1970
1971    fn into_serde_value(self) -> serde_value::Value {
1972        use serde_value::Value as SerdeValue;
1973
1974        match self {
1975            Self::Null => SerdeValue::Unit,
1976            Self::Boolean(value) => SerdeValue::Bool(value),
1977            Self::Long(value) => SerdeValue::I64(value),
1978            Self::Double(value) => SerdeValue::F64(value),
1979            Self::Bytes(value) => SerdeValue::Bytes(value),
1980            Self::String(value) => SerdeValue::String(value),
1981            Self::Array(values) => {
1982                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1983            }
1984            Self::Map(values) => SerdeValue::Map(
1985                values
1986                    .into_iter()
1987                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1988                    .collect(),
1989            ),
1990        }
1991    }
1992
1993    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1994        self.into_serde_value().deserialize_into().map_err(|error| {
1995            Error::Codec(format!(
1996                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1997            ))
1998        })
1999    }
2000}
2001
2002impl Serialize for AvroValue {
2003    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
2004    where
2005        S: Serializer,
2006    {
2007        match self {
2008            Self::Null => serializer.serialize_unit(),
2009            Self::Boolean(value) => serializer.serialize_bool(*value),
2010            Self::Long(value) => serializer.serialize_i64(*value),
2011            Self::Double(value) => serializer.serialize_f64(*value),
2012            Self::Bytes(value) => serializer.serialize_bytes(value),
2013            Self::String(value) => serializer.serialize_str(value),
2014            Self::Array(values) => {
2015                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2016                for value in values {
2017                    sequence.serialize_element(value)?;
2018                }
2019                sequence.end()
2020            }
2021            Self::Map(values) => {
2022                let mut map = serializer.serialize_map(Some(values.len()))?;
2023                for (key, value) in values {
2024                    map.serialize_entry(key, value)?;
2025                }
2026                map.end()
2027            }
2028        }
2029    }
2030}
2031
2032pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2033    let datum = avro_value_to_datum(value)?;
2034    let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2035        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2036    let mut bytes = Vec::with_capacity(datum.len() + 10);
2037    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2038    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2039    bytes.extend_from_slice(&datum);
2040    Ok(PayloadEnvelope {
2041        codec: DEFAULT_CODEC.to_string(),
2042        blob: BASE64.encode(bytes),
2043    })
2044}
2045
2046pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2047    if envelope.codec != DEFAULT_CODEC {
2048        return Err(unsupported_payload_codec(&envelope.codec));
2049    }
2050    decode_avro_value_blob(&envelope.blob)
2051}
2052
2053pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2054    let blob = match codec {
2055        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2056        other => return Err(unsupported_payload_codec(other)),
2057    };
2058
2059    Ok(PayloadEnvelope {
2060        codec: codec.to_string(),
2061        blob,
2062    })
2063}
2064
2065pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2066    match envelope.codec.as_str() {
2067        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2068        other => Err(unsupported_payload_codec(other)),
2069    }
2070}
2071
2072fn handler_type_error<T>(
2073    handler_kind: HandlerKind,
2074    handler_name: &str,
2075    value_kind: HandlerValueKind,
2076    message: impl Into<String>,
2077) -> Error {
2078    Error::HandlerType {
2079        handler_kind,
2080        handler_name: handler_name.to_string(),
2081        value_kind,
2082        rust_type: type_name::<T>(),
2083        message: message.into(),
2084    }
2085}
2086
2087fn decode_handler_input<T: DeserializeOwned>(
2088    arguments: AvroValue,
2089    handler_kind: HandlerKind,
2090    handler_name: &str,
2091) -> Result<T> {
2092    let argument = match arguments {
2093        AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2094            arguments.pop().expect("one typed handler argument")
2095        }
2096        AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2097        AvroValue::Array(arguments) => {
2098            return Err(handler_type_error::<T>(
2099                handler_kind,
2100                handler_name,
2101                HandlerValueKind::Input,
2102                format!(
2103                    "typed handlers accept one request value, but the task carried {} arguments",
2104                    arguments.len()
2105                ),
2106            ));
2107        }
2108        argument => argument,
2109    };
2110
2111    argument.deserialize().map_err(|error| {
2112        handler_type_error::<T>(
2113            handler_kind,
2114            handler_name,
2115            HandlerValueKind::Input,
2116            error.to_string(),
2117        )
2118    })
2119}
2120
2121fn encode_handler_result<T: Serialize>(
2122    result: &T,
2123    handler_kind: HandlerKind,
2124    handler_name: &str,
2125) -> Result<AvroValue> {
2126    AvroValue::from_serialize(result).map_err(|error| {
2127        handler_type_error::<T>(
2128            handler_kind,
2129            handler_name,
2130            HandlerValueKind::Result,
2131            error.to_string(),
2132        )
2133    })
2134}
2135
2136fn decode_handler_result<T: DeserializeOwned>(
2137    result: AvroValue,
2138    handler_kind: HandlerKind,
2139    handler_name: &str,
2140) -> Result<T> {
2141    result.deserialize().map_err(|error| {
2142        handler_type_error::<T>(
2143            handler_kind,
2144            handler_name,
2145            HandlerValueKind::Result,
2146            error.to_string(),
2147        )
2148    })
2149}
2150
2151#[cfg(test)]
2152fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2153    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2154}
2155
2156fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2157    validate_payload_codec(fallback_codec)?;
2158
2159    if value.is_null() {
2160        return Ok(Value::Null);
2161    }
2162
2163    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2164        return decode_blob(blob, codec);
2165    }
2166
2167    if let Some(blob) = value.as_str() {
2168        return decode_blob(blob, fallback_codec);
2169    }
2170
2171    Err(untagged_payload_value())
2172}
2173
2174fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2175    let envelope = match codec {
2176        DEFAULT_CODEC => encode_avro_value(value)?,
2177        other => return Err(unsupported_payload_codec(other)),
2178    };
2179    Ok(serde_json::to_value(envelope)?)
2180}
2181
2182fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2183    validate_payload_codec(fallback_codec)?;
2184
2185    if value.is_null() {
2186        return Ok(AvroValue::Null);
2187    }
2188
2189    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2190        validate_payload_codec(codec)?;
2191        return decode_avro_value_blob(blob);
2192    }
2193
2194    if let Some(blob) = value.as_str() {
2195        return match fallback_codec {
2196            DEFAULT_CODEC => decode_avro_value_blob(blob),
2197            other => Err(unsupported_payload_codec(other)),
2198        };
2199    }
2200
2201    Err(untagged_payload_value())
2202}
2203
2204fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2205    match value {
2206        AvroValue::Null => AvroValue::Array(Vec::new()),
2207        AvroValue::Array(_) => value,
2208        other => AvroValue::Array(vec![other]),
2209    }
2210}
2211
2212fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2213    match codec {
2214        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2215        other => Err(unsupported_payload_codec(other)),
2216    }
2217}
2218
2219fn validate_payload_codec(codec: &str) -> Result<()> {
2220    match codec {
2221        DEFAULT_CODEC => Ok(()),
2222        MISSING_TASK_PAYLOAD_CODEC => {
2223            Err(invalid_task_payload_codec("task payload_codec is missing"))
2224        }
2225        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2226        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2227            "task payload_codec must be a string",
2228        )),
2229        other => Err(unsupported_payload_codec(other)),
2230    }
2231}
2232
2233fn invalid_task_payload_codec(reason: &str) -> Error {
2234    Error::Codec(format!(
2235        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2236    ))
2237}
2238
2239fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2240    let Some(object) = value.as_object() else {
2241        return Ok(None);
2242    };
2243    if !object.contains_key("codec") && !object.contains_key("blob") {
2244        return Ok(None);
2245    }
2246
2247    let codec = object
2248        .get("codec")
2249        .and_then(Value::as_str)
2250        .ok_or_else(invalid_payload_envelope)?;
2251    validate_payload_codec(codec)?;
2252    let blob = object
2253        .get("blob")
2254        .and_then(Value::as_str)
2255        .ok_or_else(invalid_payload_envelope)?;
2256    Ok(Some((codec, blob)))
2257}
2258
2259fn invalid_payload_envelope() -> Error {
2260    Error::Codec(
2261        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2262            .to_string(),
2263    )
2264}
2265
2266fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2267    for command in commands {
2268        let Some(command) = command.as_object() else {
2269            continue;
2270        };
2271        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2272            continue;
2273        };
2274        let Some(payload_field) = workflow_command_payload_field(command_type) else {
2275            continue;
2276        };
2277
2278        if let Some(codec) = command.get("payload_codec") {
2279            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2280            validate_payload_codec(codec)?;
2281        }
2282
2283        let payload = command
2284            .get(payload_field)
2285            .ok_or_else(invalid_payload_envelope)?;
2286        if runtime_payloads::Reference::parse(payload)?.is_none() {
2287            validate_outbound_payload_envelope(payload)?;
2288        }
2289    }
2290    Ok(())
2291}
2292
2293fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2294    if commands.iter().any(|command| {
2295        command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2296            && command
2297                .get("condition_wait_occurrence_id")
2298                .and_then(Value::as_str)
2299                .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2300    }) {
2301        CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2302    } else if commands.iter().any(|command| {
2303        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2304            && command.get("attribute_types").is_some()
2305    }) {
2306        TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2307    } else if commands
2308        .iter()
2309        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2310    {
2311        MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2312    } else if commands
2313        .iter()
2314        .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2315    {
2316        CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2317    } else if commands.iter().any(|command| {
2318        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2319    }) {
2320        SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2321    } else {
2322        WORKER_PROTOCOL_VERSION
2323    }
2324}
2325
2326fn workflow_completion_protocol_version_with_message_streams(
2327    commands: &[Value],
2328    has_message_stream_metadata: bool,
2329) -> &'static str {
2330    let command_protocol = workflow_completion_protocol_version(commands);
2331    if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2332        MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2333    } else {
2334        command_protocol
2335    }
2336}
2337
2338fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2339    match command_type {
2340        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2341        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2342        "start_service_operation" => Some("request_payload"),
2343        "upsert_memo" => Some("entries"),
2344        _ => None,
2345    }
2346}
2347
2348fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2349    let Some((codec, blob)) = payload_envelope_parts(value)? else {
2350        return Err(untagged_payload_value());
2351    };
2352    validate_payload_codec(codec)?;
2353    decode_avro_value_blob(blob)?;
2354    Ok(())
2355}
2356
2357fn unsupported_payload_codec(codec: &str) -> Error {
2358    Error::Codec(format!(
2359        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2360    ))
2361}
2362
2363fn untagged_payload_value() -> Error {
2364    Error::Codec(
2365        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2366            .to_string(),
2367    )
2368}
2369
2370fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2371    let bytes = BASE64.decode(blob).map_err(|err| {
2372        Error::Codec(format!(
2373            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2374        ))
2375    })?;
2376
2377    if serde_json::from_slice::<Value>(&bytes).is_ok() {
2378        return Err(unsupported_payload_codec("json"));
2379    }
2380
2381    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2382        return Err(Error::Codec(
2383            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2384        ));
2385    }
2386
2387    let fingerprint: [u8; 8] = bytes[2..10]
2388        .try_into()
2389        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2390    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2391        return Err(Error::Codec(format!(
2392            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2393            fingerprint
2394                .iter()
2395                .map(|byte| format!("{byte:02x}"))
2396                .collect::<String>()
2397        )));
2398    }
2399
2400    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2401    // The current fingerprint selects the current immutable schema, so reader
2402    // resolution would only re-walk the same recursive union. Future retained
2403    // writer fingerprints supply a distinct reader schema in this branch.
2404    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2405    if datum_reader.truncated {
2406        return Err(Error::Codec(
2407            "invalid_payload_framing: truncated Avro Value datum".to_string(),
2408        ));
2409    }
2410    let datum = datum.map_err(|err| {
2411        Error::Codec(format!(
2412            "invalid_payload_framing: malformed Avro Value datum: {err}"
2413        ))
2414    })?;
2415    if datum_reader.remaining() != 0 {
2416        return Err(Error::Codec(format!(
2417            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2418            datum_reader.remaining()
2419        )));
2420    }
2421    avro_value_from_datum(datum)
2422}
2423
2424struct StrictAvroDatumReader<'a> {
2425    bytes: &'a [u8],
2426    offset: usize,
2427    truncated: bool,
2428}
2429
2430impl<'a> StrictAvroDatumReader<'a> {
2431    fn new(bytes: &'a [u8]) -> Self {
2432        Self {
2433            bytes,
2434            offset: 0,
2435            truncated: false,
2436        }
2437    }
2438
2439    fn remaining(&self) -> usize {
2440        self.bytes.len() - self.offset
2441    }
2442}
2443
2444impl Read for StrictAvroDatumReader<'_> {
2445    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2446        let count = buffer.len().min(self.remaining());
2447        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2448        self.offset += count;
2449        if count < buffer.len() {
2450            self.truncated = true;
2451        }
2452
2453        Ok(count)
2454    }
2455}
2456
2457fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2458    let branch = match value {
2459        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2460        AvroValue::Boolean(value) => AvroDatum::Union(
2461            1,
2462            Box::new(AvroDatum::Record(vec![(
2463                "boolean".to_string(),
2464                AvroDatum::Boolean(*value),
2465            )])),
2466        ),
2467        AvroValue::Long(value) => AvroDatum::Union(
2468            2,
2469            Box::new(AvroDatum::Record(vec![(
2470                "long".to_string(),
2471                AvroDatum::Long(*value),
2472            )])),
2473        ),
2474        AvroValue::Double(value) => {
2475            if !value.is_finite() {
2476                return Err(Error::Codec(
2477                    "non_finite_float: Avro Value doubles must be finite".to_string(),
2478                ));
2479            }
2480            AvroDatum::Union(
2481                3,
2482                Box::new(AvroDatum::Record(vec![(
2483                    "double".to_string(),
2484                    AvroDatum::Double(*value),
2485                )])),
2486            )
2487        }
2488        AvroValue::Bytes(value) => AvroDatum::Union(
2489            4,
2490            Box::new(AvroDatum::Record(vec![(
2491                "bytes".to_string(),
2492                AvroDatum::Bytes(value.clone()),
2493            )])),
2494        ),
2495        AvroValue::String(value) => AvroDatum::Union(
2496            5,
2497            Box::new(AvroDatum::Record(vec![(
2498                "string".to_string(),
2499                AvroDatum::String(value.clone()),
2500            )])),
2501        ),
2502        AvroValue::Array(values) => AvroDatum::Union(
2503            6,
2504            Box::new(AvroDatum::Record(vec![(
2505                "items".to_string(),
2506                AvroDatum::Array(
2507                    values
2508                        .iter()
2509                        .map(avro_value_to_datum)
2510                        .collect::<Result<Vec<_>>>()?,
2511                ),
2512            )])),
2513        ),
2514        AvroValue::Map(values) => AvroDatum::Union(
2515            7,
2516            Box::new(AvroDatum::Record(vec![(
2517                "entries".to_string(),
2518                AvroDatum::Array(
2519                    values
2520                        .iter()
2521                        .map(|(key, value)| {
2522                            Ok(AvroDatum::Record(vec![
2523                                ("key".to_string(), AvroDatum::String(key.clone())),
2524                                ("value".to_string(), avro_value_to_datum(value)?),
2525                            ]))
2526                        })
2527                        .collect::<Result<Vec<_>>>()?,
2528                ),
2529            )])),
2530        ),
2531    };
2532    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2533}
2534
2535fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2536    let AvroDatum::Record(mut outer) = datum else {
2537        return Err(Error::Codec(
2538            "invalid_payload_framing: datum is not a Value record".to_string(),
2539        ));
2540    };
2541    let (_, branch) = outer
2542        .pop()
2543        .filter(|(name, _)| name == "value")
2544        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2545    let AvroDatum::Union(_, branch) = branch else {
2546        return Err(Error::Codec(
2547            "invalid_payload_framing: invalid Value union".to_string(),
2548        ));
2549    };
2550    match *branch {
2551        AvroDatum::Null => Ok(AvroValue::Null),
2552        AvroDatum::Record(mut fields) => {
2553            let (name, value) = fields.pop().ok_or_else(|| {
2554                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2555            })?;
2556            match (name.as_str(), value) {
2557                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2558                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2559                ("double", AvroDatum::Double(value)) if value.is_finite() => {
2560                    Ok(AvroValue::Double(value))
2561                }
2562                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2563                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2564                ("items", AvroDatum::Array(values)) => values
2565                    .into_iter()
2566                    .map(avro_value_from_datum)
2567                    .collect::<Result<Vec<_>>>()
2568                    .map(AvroValue::Array),
2569                ("entries", AvroDatum::Map(values)) => values
2570                    .into_iter()
2571                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2572                    .collect::<Result<BTreeMap<_, _>>>()
2573                    .map(AvroValue::Map),
2574                _ => Err(Error::Codec(
2575                    "invalid_payload_framing: unknown Value branch".to_string(),
2576                )),
2577            }
2578        }
2579        _ => Err(Error::Codec(
2580            "invalid_payload_framing: invalid Value branch".to_string(),
2581        )),
2582    }
2583}
2584
2585fn avro_value_schema() -> Result<&'static Schema> {
2586    match AVRO_VALUE_SCHEMA.get_or_init(|| {
2587        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2588            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2589    }) {
2590        Ok(schema) => Ok(schema),
2591        Err(message) => Err(Error::Codec(message.clone())),
2592    }
2593}
2594
2595fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2596    match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2597        // Apache Avro's Value::Map uses a randomized HashMap. Arrays and maps
2598        // have the same block representation when each ordered array record
2599        // contains the map key followed by its value, so this encoding-only
2600        // adaptation lets the official encoder retain BTreeMap wire order.
2601        let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2602            .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2603        let entries_schema = schema
2604            .pointer_mut("/fields/0/type/7/fields/0/type")
2605            .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2606        if *entries_schema != json!({"type": "map", "values": "Value"}) {
2607            return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2608        }
2609        *entries_schema = json!({
2610            "type": "array",
2611            "items": {
2612                "type": "record",
2613                "name": "MapEntry",
2614                "fields": [
2615                    {"name": "key", "type": "string"},
2616                    {"name": "value", "type": "Value"}
2617                ]
2618            }
2619        });
2620        Schema::parse_str(&schema.to_string())
2621            .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2622    }) {
2623        Ok(schema) => Ok(schema),
2624        Err(message) => Err(Error::Codec(message.clone())),
2625    }
2626}
2627
2628#[derive(Clone, Debug)]
2629pub struct Client {
2630    http: reqwest::Client,
2631    base_url: String,
2632    token: Option<String>,
2633    control_token: Option<String>,
2634    worker_token: Option<String>,
2635    namespace: String,
2636    max_external_payload_bytes: usize,
2637    worker_storage_admission: Option<WorkerStorageAdmission>,
2638    runtime_upload_policy: Arc<Mutex<runtime_uploads::PolicyCache>>,
2639}
2640
2641impl Client {
2642    pub fn new(base_url: impl Into<String>) -> Result<Self> {
2643        Self::builder(base_url).build()
2644    }
2645
2646    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2647        ClientBuilder {
2648            base_url: base_url.into(),
2649            token: None,
2650            control_token: None,
2651            worker_token: None,
2652            namespace: "default".to_string(),
2653            timeout: Duration::from_secs(60),
2654            max_external_payload_bytes: 64 * 1024 * 1024,
2655        }
2656    }
2657
2658    pub async fn health(&self) -> Result<Value> {
2659        self.request_json(
2660            reqwest::Method::GET,
2661            "/health",
2662            RequestProtocol::ControlPlane,
2663            Option::<&Value>::None,
2664        )
2665        .await
2666    }
2667
2668    pub async fn cluster_info(&self) -> Result<Value> {
2669        self.request_json(
2670            reqwest::Method::GET,
2671            "/cluster/info",
2672            RequestProtocol::ControlPlane,
2673            Option::<&Value>::None,
2674        )
2675        .await
2676    }
2677
2678    pub async fn start_workflow<T: Serialize>(
2679        &self,
2680        workflow_type: &str,
2681        task_queue: &str,
2682        workflow_id: &str,
2683        input: T,
2684    ) -> Result<WorkflowHandle> {
2685        self.start_workflow_with_options(
2686            workflow_type,
2687            task_queue,
2688            workflow_id,
2689            WorkflowStartOptions::default(),
2690            input,
2691        )
2692        .await
2693    }
2694
2695    /// Start a workflow with explicit server-enforced execution and run
2696    /// deadlines.
2697    pub async fn start_workflow_with_options<T: Serialize>(
2698        &self,
2699        workflow_type: &str,
2700        task_queue: &str,
2701        workflow_id: &str,
2702        options: WorkflowStartOptions,
2703        input: T,
2704    ) -> Result<WorkflowHandle> {
2705        options.validate()?;
2706        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2707        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2708        let body = json!({
2709            "workflow_id": workflow_id,
2710            "workflow_type": workflow_type,
2711            "task_queue": task_queue,
2712            "input": input_envelope,
2713            "execution_timeout_seconds": options.execution_timeout_seconds,
2714            "run_timeout_seconds": options.run_timeout_seconds
2715        });
2716
2717        let data: Value = self
2718            .request_json(
2719                reqwest::Method::POST,
2720                "/workflows",
2721                RequestProtocol::ControlPlane,
2722                Some(&body),
2723            )
2724            .await?;
2725
2726        Ok(WorkflowHandle {
2727            client: self.clone(),
2728            workflow_id: data
2729                .get("workflow_id")
2730                .and_then(Value::as_str)
2731                .unwrap_or(workflow_id)
2732                .to_string(),
2733            run_id: data
2734                .get("run_id")
2735                .and_then(Value::as_str)
2736                .map(str::to_string),
2737            workflow_type: data
2738                .get("workflow_type")
2739                .and_then(Value::as_str)
2740                .unwrap_or(workflow_type)
2741                .to_string(),
2742        })
2743    }
2744
2745    pub async fn signal_workflow<T: Serialize>(
2746        &self,
2747        workflow_id: &str,
2748        signal_name: &str,
2749        input: T,
2750    ) -> Result<Value> {
2751        self.signal_workflow_target(workflow_id, None, signal_name, input)
2752            .await
2753    }
2754
2755    /// Append one idempotently identified item to an instance-scoped input stream.
2756    pub async fn append_message_stream<T: Serialize>(
2757        &self,
2758        workflow_id: &str,
2759        stream_name: &str,
2760        message_id: &str,
2761        input: T,
2762    ) -> Result<Value> {
2763        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2764        let body = json!({
2765            "message_id": message_id,
2766            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2767        });
2768        self.request_json(
2769            reqwest::Method::POST,
2770            &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2771            RequestProtocol::ControlPlane,
2772            Some(&body),
2773        )
2774        .await
2775    }
2776
2777    /// Signal only if `run_id` is still the current run for this instance.
2778    pub async fn signal_workflow_run<T: Serialize>(
2779        &self,
2780        workflow_id: &str,
2781        run_id: &str,
2782        signal_name: &str,
2783        input: T,
2784    ) -> Result<Value> {
2785        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2786            .await
2787    }
2788
2789    async fn signal_workflow_target<T: Serialize>(
2790        &self,
2791        workflow_id: &str,
2792        run_id: Option<&str>,
2793        signal_name: &str,
2794        input: T,
2795    ) -> Result<Value> {
2796        validate_user_signal_name(signal_name)?;
2797        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2798        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2799        let body = json!({
2800            "input": input_envelope
2801        });
2802        let path = match run_id {
2803            Some(run_id) => {
2804                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2805            }
2806            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2807        };
2808        self.request_json(
2809            reqwest::Method::POST,
2810            &path,
2811            RequestProtocol::ControlPlane,
2812            Some(&body),
2813        )
2814        .await
2815    }
2816
2817    /// Close the current run as cancelled immediately.
2818    ///
2819    /// Server revokes open tasks and timers without resuming workflow code for
2820    /// cleanup. Embedded Laravel's cooperative `requestCancellation()` is a
2821    /// separate capability, not yet available in service mode.
2822    pub async fn cancel_workflow(
2823        &self,
2824        workflow_id: &str,
2825        options: WorkflowCommandOptions,
2826    ) -> Result<WorkflowCommandResult> {
2827        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2828            .await
2829    }
2830
2831    /// Close the selected run as cancelled, only if `run_id` is still current.
2832    pub async fn cancel_workflow_run(
2833        &self,
2834        workflow_id: &str,
2835        run_id: &str,
2836        options: WorkflowCommandOptions,
2837    ) -> Result<WorkflowCommandResult> {
2838        self.workflow_command(
2839            workflow_id,
2840            Some(run_id),
2841            WorkflowCommandKind::Cancel,
2842            options,
2843        )
2844        .await
2845    }
2846
2847    /// Forcefully terminate the current run for an instance.
2848    pub async fn terminate_workflow(
2849        &self,
2850        workflow_id: &str,
2851        options: WorkflowCommandOptions,
2852    ) -> Result<WorkflowCommandResult> {
2853        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2854            .await
2855    }
2856
2857    /// Forcefully terminate only if `run_id` is still current.
2858    pub async fn terminate_workflow_run(
2859        &self,
2860        workflow_id: &str,
2861        run_id: &str,
2862        options: WorkflowCommandOptions,
2863    ) -> Result<WorkflowCommandResult> {
2864        self.workflow_command(
2865            workflow_id,
2866            Some(run_id),
2867            WorkflowCommandKind::Terminate,
2868            options,
2869        )
2870        .await
2871    }
2872
2873    /// Continue a failed run from its recorded activity failure boundary.
2874    pub async fn redrive_workflow_run(
2875        &self,
2876        workflow_id: &str,
2877        failed_run_id: &str,
2878        request_id: Option<&str>,
2879    ) -> Result<WorkflowRedriveResult> {
2880        let body = request_id
2881            .map(|id| json!({"request_id": id}))
2882            .unwrap_or_else(|| json!({}));
2883        let data: Value = self
2884            .request_json(
2885                reqwest::Method::POST,
2886                &format!("/workflows/{workflow_id}/runs/{failed_run_id}/redrive"),
2887                RequestProtocol::ControlPlane,
2888                Some(&body),
2889            )
2890            .await?;
2891        if data.get("command_status").and_then(Value::as_str) != Some("accepted")
2892            || data.get("outcome").and_then(Value::as_str) != Some("redriven")
2893        {
2894            return Err(Error::Codec(
2895                "redrive response was not accepted".to_string(),
2896            ));
2897        }
2898        if data.get("workflow_id").and_then(Value::as_str) != Some(workflow_id)
2899            || data.get("continued_from_run_id").and_then(Value::as_str) != Some(failed_run_id)
2900        {
2901            return Err(Error::Codec(
2902                "redrive response does not match the requested source run".to_string(),
2903            ));
2904        }
2905        let run_id = data
2906            .get("run_id")
2907            .and_then(Value::as_str)
2908            .filter(|id| !id.is_empty() && *id != failed_run_id)
2909            .ok_or_else(|| {
2910                Error::Codec("redrive response is missing a distinct successor run_id".to_string())
2911            })?;
2912        let resume_step_sequence = data
2913            .get("resume_step_sequence")
2914            .and_then(Value::as_u64)
2915            .filter(|sequence| *sequence > 0)
2916            .ok_or_else(|| {
2917                Error::Codec("redrive response is missing resume_step_sequence".to_string())
2918            })?;
2919        Ok(WorkflowRedriveResult {
2920            workflow_id: workflow_id.to_string(),
2921            source_run_id: failed_run_id.to_string(),
2922            run_id: run_id.to_string(),
2923            resume_step_sequence,
2924            raw: data,
2925        })
2926    }
2927
2928    async fn workflow_command(
2929        &self,
2930        workflow_id: &str,
2931        run_id: Option<&str>,
2932        command: WorkflowCommandKind,
2933        options: WorkflowCommandOptions,
2934    ) -> Result<WorkflowCommandResult> {
2935        let path = match run_id {
2936            Some(run_id) => format!(
2937                "/workflows/{workflow_id}/runs/{run_id}/{}",
2938                command.as_str()
2939            ),
2940            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2941        };
2942        let data = match self
2943            .request_json(
2944                reqwest::Method::POST,
2945                &path,
2946                RequestProtocol::ControlPlane,
2947                Some(&options),
2948            )
2949            .await
2950        {
2951            Ok(data) => data,
2952            Err(Error::Http { status, body }) => {
2953                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2954                    command,
2955                    status,
2956                    body,
2957                    workflow_id,
2958                    run_id,
2959                )));
2960            }
2961            Err(error) => return Err(error),
2962        };
2963
2964        Ok(workflow_command_result(command, data, workflow_id, run_id))
2965    }
2966
2967    /// Execute a named, read-only query against a running or completed workflow.
2968    ///
2969    /// Arguments and results use the platform payload envelope. Server and
2970    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
2971    /// reason, HTTP status, and original response body.
2972    pub async fn query_workflow<T: Serialize>(
2973        &self,
2974        workflow_id: &str,
2975        query_name: &str,
2976        input: T,
2977    ) -> Result<Value> {
2978        self.query_workflow_target(workflow_id, None, query_name, input)
2979            .await
2980    }
2981
2982    /// Query only if `run_id` is still current, preventing accidental retargeting.
2983    pub async fn query_workflow_run<T: Serialize>(
2984        &self,
2985        workflow_id: &str,
2986        run_id: &str,
2987        query_name: &str,
2988        input: T,
2989    ) -> Result<Value> {
2990        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2991            .await
2992    }
2993
2994    /// Query a workflow and return the lossless fixed Avro Value result.
2995    pub async fn query_workflow_avro_value<T: Serialize>(
2996        &self,
2997        workflow_id: &str,
2998        query_name: &str,
2999        input: T,
3000    ) -> Result<AvroValue> {
3001        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
3002            .await
3003    }
3004
3005    /// Query a selected run and return the lossless fixed Avro Value result.
3006    pub async fn query_workflow_run_avro_value<T: Serialize>(
3007        &self,
3008        workflow_id: &str,
3009        run_id: &str,
3010        query_name: &str,
3011        input: T,
3012    ) -> Result<AvroValue> {
3013        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
3014            .await
3015    }
3016
3017    async fn query_workflow_avro_value_target<T: Serialize>(
3018        &self,
3019        workflow_id: &str,
3020        run_id: Option<&str>,
3021        query_name: &str,
3022        input: T,
3023    ) -> Result<AvroValue> {
3024        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3025        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
3026        let path = match run_id {
3027            Some(run_id) => {
3028                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3029            }
3030            None => format!("/workflows/{workflow_id}/query/{query_name}"),
3031        };
3032        let response: Value = match self
3033            .request_json(
3034                reqwest::Method::POST,
3035                &path,
3036                RequestProtocol::ControlPlane,
3037                Some(&body),
3038            )
3039            .await
3040        {
3041            Ok(response) => response,
3042            Err(Error::Http { status, body }) => {
3043                return Err(Error::QueryFailed(query_failure(status, body)));
3044            }
3045            Err(error) => return Err(error),
3046        };
3047
3048        let envelope = response
3049            .get("result_envelope")
3050            .filter(|envelope| !envelope.is_null())
3051            .ok_or_else(|| {
3052                Error::Codec(
3053                    "missing_payload_envelope: typed query result requires result_envelope"
3054                        .to_string(),
3055                )
3056            })?;
3057        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3058    }
3059
3060    async fn query_workflow_target<T: Serialize>(
3061        &self,
3062        workflow_id: &str,
3063        run_id: Option<&str>,
3064        query_name: &str,
3065        input: T,
3066    ) -> Result<Value> {
3067        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3068        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
3069        let body = json!({
3070            "input": input_envelope
3071        });
3072        let path = match run_id {
3073            Some(run_id) => {
3074                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3075            }
3076            None => format!("/workflows/{workflow_id}/query/{query_name}"),
3077        };
3078        let response: Value = match self
3079            .request_json(
3080                reqwest::Method::POST,
3081                &path,
3082                RequestProtocol::ControlPlane,
3083                Some(&body),
3084            )
3085            .await
3086        {
3087            Ok(response) => response,
3088            Err(Error::Http { status, body }) => {
3089                return Err(Error::QueryFailed(query_failure(status, body)));
3090            }
3091            Err(error) => return Err(error),
3092        };
3093
3094        if let Some(envelope) = response
3095            .get("result_envelope")
3096            .filter(|envelope| !envelope.is_null())
3097        {
3098            return decode_wire_value(envelope, DEFAULT_CODEC);
3099        }
3100
3101        Ok(response.get("result").cloned().unwrap_or(Value::Null))
3102    }
3103
3104    /// Send a synchronous update using fixed Avro Value arguments.
3105    pub async fn update_workflow<T: Serialize>(
3106        &self,
3107        workflow_id: &str,
3108        update_name: &str,
3109        input: T,
3110        request_id: Option<&str>,
3111    ) -> Result<Value> {
3112        let response = self
3113            .update_workflow_response(workflow_id, update_name, input, request_id)
3114            .await?;
3115        if let Some(envelope) = response
3116            .get("result_envelope")
3117            .filter(|envelope| !envelope.is_null())
3118        {
3119            return decode_wire_value(envelope, DEFAULT_CODEC);
3120        }
3121        Ok(response.get("result").cloned().unwrap_or(response))
3122    }
3123
3124    /// Send a synchronous update and retain a bytes-capable Avro result.
3125    pub async fn update_workflow_avro_value<T: Serialize>(
3126        &self,
3127        workflow_id: &str,
3128        update_name: &str,
3129        input: T,
3130        request_id: Option<&str>,
3131    ) -> Result<AvroValue> {
3132        let response = self
3133            .update_workflow_response(workflow_id, update_name, input, request_id)
3134            .await?;
3135        let envelope = response
3136            .get("result_envelope")
3137            .filter(|envelope| !envelope.is_null())
3138            .ok_or_else(|| {
3139                Error::Codec(
3140                    "missing_payload_envelope: typed update result requires result_envelope"
3141                        .to_string(),
3142                )
3143            })?;
3144        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3145    }
3146
3147    async fn update_workflow_response<T: Serialize>(
3148        &self,
3149        workflow_id: &str,
3150        update_name: &str,
3151        input: T,
3152        request_id: Option<&str>,
3153    ) -> Result<Value> {
3154        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3155        let mut body = json!({
3156            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3157            "wait_for": "completed",
3158        });
3159        if let Some(request_id) = request_id {
3160            body["request_id"] = json!(request_id);
3161        }
3162        self.request_json(
3163            reqwest::Method::POST,
3164            &format!("/workflows/{workflow_id}/update/{update_name}"),
3165            RequestProtocol::ControlPlane,
3166            Some(&body),
3167        )
3168        .await
3169    }
3170
3171    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3172        let path = format!("/workflows/{workflow_id}");
3173        let mut data: WorkflowDescription = self
3174            .request_json(
3175                reqwest::Method::GET,
3176                &path,
3177                RequestProtocol::ControlPlane,
3178                Option::<&Value>::None,
3179            )
3180            .await?;
3181        data.decode_payloads()?;
3182        Ok(data)
3183    }
3184
3185    /// Describe one selected run, including historical terminal runs.
3186    pub async fn describe_workflow_run(
3187        &self,
3188        workflow_id: &str,
3189        run_id: &str,
3190    ) -> Result<WorkflowDescription> {
3191        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3192        let mut data: WorkflowDescription = self
3193            .request_json(
3194                reqwest::Method::GET,
3195                &path,
3196                RequestProtocol::ControlPlane,
3197                Option::<&Value>::None,
3198            )
3199            .await?;
3200        data.decode_payloads()?;
3201        Ok(data)
3202    }
3203
3204    fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3205        let mut path = format!(
3206            "/workflows/{}/runs/{}/streams",
3207            percent_encode_path_segment(workflow_id),
3208            percent_encode_path_segment(run_id),
3209        );
3210        if let Some(stream_name) = stream_name {
3211            path.push('/');
3212            path.push_str(&percent_encode_path_segment(stream_name));
3213        }
3214        path
3215    }
3216
3217    /// List the run-scoped output streams already opened by a workflow.
3218    pub async fn list_workflow_streams(
3219        &self,
3220        workflow_id: &str,
3221        run_id: &str,
3222    ) -> Result<Vec<WorkflowStreamDescription>> {
3223        let response: WorkflowStreamListResponse = self
3224            .request_json(
3225                reqwest::Method::GET,
3226                &Self::workflow_stream_path(workflow_id, run_id, None),
3227                RequestProtocol::ControlPlane,
3228                Option::<&Value>::None,
3229            )
3230            .await?;
3231        Ok(response.streams)
3232    }
3233
3234    /// Describe stream lifecycle, offsets, pending count, and terminal error.
3235    pub async fn describe_workflow_stream(
3236        &self,
3237        workflow_id: &str,
3238        run_id: &str,
3239        stream_name: &str,
3240    ) -> Result<WorkflowStreamDescription> {
3241        let response: WorkflowStreamDescriptionResponse = self
3242            .request_json(
3243                reqwest::Method::GET,
3244                &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3245                RequestProtocol::ControlPlane,
3246                Option::<&Value>::None,
3247            )
3248            .await?;
3249        Ok(response.stream)
3250    }
3251
3252    /// Read one bounded page beginning at a zero-based offset.
3253    ///
3254    /// Delivery is at least once: persist `next_offset` only after processing
3255    /// the page. The future is cancellation-safe; dropping it cancels the
3256    /// in-flight request. Long polling is capped at 60 seconds by the SDK and
3257    /// service contract.
3258    pub async fn subscribe_workflow_stream(
3259        &self,
3260        workflow_id: &str,
3261        run_id: &str,
3262        stream_name: &str,
3263        from_offset: u64,
3264        max_items: usize,
3265        wait: Duration,
3266    ) -> Result<WorkflowStreamPage> {
3267        let max_items = max_items.clamp(1, 500);
3268        let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3269        let path = format!(
3270            "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3271            Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3272        );
3273        let response: WorkflowStreamPageResponse = self
3274            .request_json_with_timeout(
3275                reqwest::Method::GET,
3276                &path,
3277                RequestProtocol::ControlPlane,
3278                Option::<&Value>::None,
3279                Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3280            )
3281            .await?;
3282
3283        let items = response
3284            .items
3285            .into_iter()
3286            .map(|raw| {
3287                let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3288                let envelope = raw.get("payload").cloned();
3289                let payload = envelope
3290                    .as_ref()
3291                    .filter(|value| value.get("blob").is_some())
3292                    .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3293                    .transpose()?
3294                    .map(AvroValue::into_json)
3295                    .transpose()?;
3296                Ok(WorkflowStreamItem {
3297                    offset,
3298                    payload,
3299                    payload_envelope: envelope,
3300                    payload_reference: raw
3301                        .get("payload_reference")
3302                        .and_then(Value::as_str)
3303                        .map(str::to_string),
3304                    payload_codec: raw
3305                        .get("payload_codec")
3306                        .and_then(Value::as_str)
3307                        .map(str::to_string),
3308                    idempotency_key: raw
3309                        .get("idempotency_key")
3310                        .and_then(Value::as_str)
3311                        .map(str::to_string),
3312                    item_type: raw
3313                        .get("item_type")
3314                        .and_then(Value::as_str)
3315                        .map(str::to_string),
3316                    content_type: raw
3317                        .get("content_type")
3318                        .and_then(Value::as_str)
3319                        .map(str::to_string),
3320                    origin: raw
3321                        .get("origin")
3322                        .and_then(Value::as_str)
3323                        .map(str::to_string),
3324                    origin_reference: raw
3325                        .get("origin_reference")
3326                        .and_then(Value::as_str)
3327                        .map(str::to_string),
3328                    emitted_at: raw
3329                        .get("emitted_at")
3330                        .and_then(Value::as_str)
3331                        .map(str::to_string),
3332                    raw,
3333                })
3334            })
3335            .collect::<Result<Vec<_>>>()?;
3336        Ok(WorkflowStreamPage {
3337            stream: response.stream,
3338            items,
3339            next_offset: response.next_offset,
3340            terminal: response.terminal,
3341        })
3342    }
3343
3344    /// Append inline Avro envelopes or opaque external payload references.
3345    pub async fn append_workflow_stream(
3346        &self,
3347        workflow_id: &str,
3348        run_id: &str,
3349        stream_name: &str,
3350        items: &[WorkflowStreamAppendItem],
3351        max_pending_items: Option<u64>,
3352    ) -> Result<WorkflowStreamAppendResult> {
3353        if items.is_empty() {
3354            return Err(Error::Codec(
3355                "workflow_stream_items_empty: append requires at least one item".to_string(),
3356            ));
3357        }
3358        let mut body = json!({
3359            "items": items
3360                .iter()
3361                .map(|item| item.wire_value(None))
3362                .collect::<Vec<_>>(),
3363        });
3364        if let Some(max_pending_items) = max_pending_items {
3365            if max_pending_items == 0 {
3366                return Err(Error::Codec(
3367                    "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3368                        .to_string(),
3369                ));
3370            }
3371            body["max_pending_items"] = json!(max_pending_items);
3372        }
3373        let response: WorkflowStreamAppendResponse = self
3374            .request_json(
3375                reqwest::Method::POST,
3376                &format!(
3377                    "{}/items",
3378                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3379                ),
3380                RequestProtocol::ControlPlane,
3381                Some(&body),
3382            )
3383            .await?;
3384        Ok(WorkflowStreamAppendResult {
3385            stream: response.stream,
3386            accepted_offsets: response.accepted_offsets,
3387            accepted: response.accepted,
3388            deduped: response.deduped,
3389        })
3390    }
3391
3392    /// Close a stream, or mark it errored when `error_reason` is supplied.
3393    pub async fn close_workflow_stream(
3394        &self,
3395        workflow_id: &str,
3396        run_id: &str,
3397        stream_name: &str,
3398        error_reason: Option<&str>,
3399        retention_seconds: Option<u64>,
3400    ) -> Result<WorkflowStreamDescription> {
3401        let mut body = json!({});
3402        if let Some(error_reason) = error_reason {
3403            body["error_reason"] = json!(error_reason);
3404        }
3405        if let Some(retention_seconds) = retention_seconds {
3406            if retention_seconds == 0 {
3407                return Err(Error::Codec(
3408                    "workflow_stream_retention_invalid: retention_seconds must be positive"
3409                        .to_string(),
3410                ));
3411            }
3412            body["retention_seconds"] = json!(retention_seconds);
3413        }
3414        let response: WorkflowStreamDescriptionResponse = self
3415            .request_json(
3416                reqwest::Method::POST,
3417                &format!(
3418                    "{}/close",
3419                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3420                ),
3421                RequestProtocol::ControlPlane,
3422                Some(&body),
3423            )
3424            .await?;
3425        Ok(response.stream)
3426    }
3427
3428    pub async fn register_worker(
3429        &self,
3430        worker_id: &str,
3431        task_queue: &str,
3432        supported_workflow_types: Vec<String>,
3433        supported_activity_types: Vec<String>,
3434        max_concurrent_workflow_tasks: usize,
3435        max_concurrent_activity_tasks: usize,
3436    ) -> Result<RegisterWorkerResponse> {
3437        self.register_worker_with_capabilities(
3438            worker_id,
3439            task_queue,
3440            supported_workflow_types,
3441            supported_activity_types,
3442            max_concurrent_workflow_tasks,
3443            max_concurrent_activity_tasks,
3444            Vec::new(),
3445        )
3446        .await
3447    }
3448
3449    /// Register a worker and explicitly advertise additive worker capabilities.
3450    pub async fn register_worker_with_capabilities(
3451        &self,
3452        worker_id: &str,
3453        task_queue: &str,
3454        supported_workflow_types: Vec<String>,
3455        supported_activity_types: Vec<String>,
3456        max_concurrent_workflow_tasks: usize,
3457        max_concurrent_activity_tasks: usize,
3458        capabilities: Vec<String>,
3459    ) -> Result<RegisterWorkerResponse> {
3460        self.register_worker_with_command_contracts(
3461            worker_id,
3462            task_queue,
3463            supported_workflow_types,
3464            supported_activity_types,
3465            max_concurrent_workflow_tasks,
3466            max_concurrent_activity_tasks,
3467            capabilities,
3468            Value::Object(serde_json::Map::new()),
3469        )
3470        .await
3471    }
3472
3473    /// Register a worker and advertise its named query and update handlers.
3474    ///
3475    /// This Rust SDK cannot execute synchronous pre-accept update validation,
3476    /// so a workflow contract with a non-empty or malformed
3477    /// `update_validators` declaration returns
3478    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
3479    #[allow(clippy::too_many_arguments)]
3480    pub async fn register_worker_with_command_contracts(
3481        &self,
3482        worker_id: &str,
3483        task_queue: &str,
3484        supported_workflow_types: Vec<String>,
3485        supported_activity_types: Vec<String>,
3486        max_concurrent_workflow_tasks: usize,
3487        max_concurrent_activity_tasks: usize,
3488        capabilities: Vec<String>,
3489        workflow_command_contracts: Value,
3490    ) -> Result<RegisterWorkerResponse> {
3491        self.register_worker_with_definition_fingerprints(
3492            worker_id,
3493            task_queue,
3494            supported_workflow_types,
3495            supported_activity_types,
3496            max_concurrent_workflow_tasks,
3497            max_concurrent_activity_tasks,
3498            capabilities,
3499            workflow_command_contracts,
3500            None,
3501        )
3502        .await
3503    }
3504
3505    #[allow(clippy::too_many_arguments)]
3506    async fn register_worker_with_definition_fingerprints(
3507        &self,
3508        worker_id: &str,
3509        task_queue: &str,
3510        supported_workflow_types: Vec<String>,
3511        supported_activity_types: Vec<String>,
3512        max_concurrent_workflow_tasks: usize,
3513        max_concurrent_activity_tasks: usize,
3514        capabilities: Vec<String>,
3515        workflow_command_contracts: Value,
3516        workflow_definition_fingerprints: Option<BTreeMap<String, String>>,
3517    ) -> Result<RegisterWorkerResponse> {
3518        if let Some(contracts) = workflow_command_contracts.as_object() {
3519            for (workflow_type, contract) in contracts {
3520                let Some(update_validators) = contract.get("update_validators") else {
3521                    continue;
3522                };
3523                if !update_validators
3524                    .as_array()
3525                    .is_some_and(|validators| validators.is_empty())
3526                {
3527                    return Err(Error::UnsupportedUpdateValidators {
3528                        workflow_type: workflow_type.clone(),
3529                    });
3530                }
3531            }
3532        }
3533
3534        let mut body = json!({
3535            "worker_id": worker_id,
3536            "task_queue": task_queue,
3537            "runtime": "rust",
3538            "sdk_version": SDK_VERSION,
3539            "supported_workflow_types": supported_workflow_types,
3540            "supported_activity_types": supported_activity_types,
3541            "capabilities": capabilities,
3542            "capability_manifest": portable_worker_affinity_capability_manifest(),
3543            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3544            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3545        });
3546        if workflow_command_contracts
3547            .as_object()
3548            .is_some_and(|contracts| !contracts.is_empty())
3549        {
3550            body["workflow_command_contracts"] = workflow_command_contracts;
3551        }
3552        if let Some(fingerprints) = workflow_definition_fingerprints {
3553            body["workflow_definition_fingerprints"] = json!(fingerprints);
3554        }
3555
3556        self.request_json(
3557            reqwest::Method::POST,
3558            "/worker/register",
3559            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3560            Some(&body),
3561        )
3562        .await
3563    }
3564
3565    /// Gracefully remove one worker's registration through the worker plane.
3566    ///
3567    /// This operation is separate from operator-facing worker management. It
3568    /// uses worker-protocol authentication and returns the server's lease
3569    /// recovery result.
3570    pub async fn deregister_worker_registration(
3571        &self,
3572        worker_id: &str,
3573    ) -> Result<WorkerDeregistrationEnvelope> {
3574        let path = format!(
3575            "/worker/registrations/{}",
3576            percent_encode_path_segment(worker_id)
3577        );
3578        self.request_json(
3579            reqwest::Method::DELETE,
3580            &path,
3581            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3582            Option::<&Value>::None,
3583        )
3584        .await
3585    }
3586
3587    /// Long-poll for an ephemeral, read-only workflow query task.
3588    pub async fn poll_query_task(
3589        &self,
3590        worker_id: &str,
3591        task_queue: &str,
3592        timeout: Duration,
3593    ) -> Result<Option<QueryTask>> {
3594        Ok(self
3595            .poll_query_task_response(worker_id, task_queue, timeout)
3596            .await?
3597            .task)
3598    }
3599
3600    /// Poll a query task while preserving server stop and drain metadata.
3601    pub async fn poll_query_task_response(
3602        &self,
3603        worker_id: &str,
3604        task_queue: &str,
3605        timeout: Duration,
3606    ) -> Result<PollQueryTaskResponse> {
3607        let poll_request_id = unique_request_id("rust-query-poll");
3608        self.poll_query_task_response_with_request_id(
3609            worker_id,
3610            task_queue,
3611            timeout,
3612            &poll_request_id,
3613            1,
3614        )
3615        .await
3616    }
3617
3618    async fn poll_query_task_response_with_request_id(
3619        &self,
3620        worker_id: &str,
3621        task_queue: &str,
3622        timeout: Duration,
3623        poll_request_id: &str,
3624        transport_retries: usize,
3625    ) -> Result<PollQueryTaskResponse> {
3626        let timeout_seconds = long_poll_timeout_seconds(timeout);
3627        let body = json!({
3628            "worker_id": worker_id,
3629            "task_queue": task_queue,
3630            "poll_request_id": poll_request_id,
3631            "timeout_seconds": timeout_seconds,
3632        });
3633        self.poll_request_json(
3634            "/worker/query-tasks/poll",
3635            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3636            &body,
3637            timeout + Duration::from_secs(5),
3638            transport_retries,
3639        )
3640        .await
3641    }
3642
3643    /// Complete a query task without appending workflow history.
3644    pub async fn complete_query_task<T: Serialize>(
3645        &self,
3646        query_task_id: &str,
3647        lease_owner: &str,
3648        query_task_attempt: u64,
3649        result: T,
3650        codec: &str,
3651    ) -> Result<Value> {
3652        let typed_result = AvroValue::from_serialize(&result)?;
3653        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3654        self.complete_query_task_with_envelope(
3655            query_task_id,
3656            lease_owner,
3657            query_task_attempt,
3658            typed_result.into_json()?,
3659            result_envelope,
3660        )
3661        .await
3662    }
3663
3664    async fn complete_query_task_with_envelope(
3665        &self,
3666        query_task_id: &str,
3667        lease_owner: &str,
3668        query_task_attempt: u64,
3669        result: Value,
3670        result_envelope: Value,
3671    ) -> Result<Value> {
3672        let body = json!({
3673            "lease_owner": lease_owner,
3674            "query_task_attempt": query_task_attempt,
3675            "result": result,
3676            "result_envelope": result_envelope,
3677        });
3678        let path = format!("/worker/query-tasks/{query_task_id}/complete");
3679        let response = self
3680            .request_json(
3681                reqwest::Method::POST,
3682                &path,
3683                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3684                Some(&body),
3685            )
3686            .await;
3687        query_task_response(response)
3688    }
3689
3690    /// Report a stable machine-readable query-task failure.
3691    pub async fn fail_query_task(
3692        &self,
3693        query_task_id: &str,
3694        lease_owner: &str,
3695        query_task_attempt: u64,
3696        message: impl Into<String>,
3697        reason: impl Into<String>,
3698        failure_type: impl Into<String>,
3699    ) -> Result<Value> {
3700        let body = json!({
3701            "lease_owner": lease_owner,
3702            "query_task_attempt": query_task_attempt,
3703            "failure": {
3704                "message": message.into(),
3705                "reason": reason.into(),
3706                "type": failure_type.into(),
3707            }
3708        });
3709        let path = format!("/worker/query-tasks/{query_task_id}/fail");
3710        let response = self
3711            .request_json(
3712                reqwest::Method::POST,
3713                &path,
3714                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3715                Some(&body),
3716            )
3717            .await;
3718        query_task_response(response)
3719    }
3720
3721    pub async fn heartbeat_worker(
3722        &self,
3723        worker_id: &str,
3724        workflow_available: usize,
3725        activity_available: usize,
3726    ) -> Result<Value> {
3727        let body = json!({
3728            "worker_id": worker_id,
3729            "task_slots": {
3730                "workflow_available": workflow_available,
3731                "activity_available": activity_available
3732            },
3733            "process_metrics": {
3734                "process_id": std::process::id(),
3735                "process_uptime_seconds": 0
3736            }
3737        });
3738
3739        self.request_json(
3740            reqwest::Method::POST,
3741            "/worker/heartbeat",
3742            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3743            Some(&body),
3744        )
3745        .await
3746    }
3747
3748    pub async fn poll_workflow_task(
3749        &self,
3750        worker_id: &str,
3751        task_queue: &str,
3752        timeout: Duration,
3753    ) -> Result<Option<WorkflowTask>> {
3754        Ok(self
3755            .poll_workflow_task_response(worker_id, task_queue, timeout)
3756            .await?
3757            .task)
3758    }
3759
3760    pub async fn poll_workflow_task_response(
3761        &self,
3762        worker_id: &str,
3763        task_queue: &str,
3764        timeout: Duration,
3765    ) -> Result<PollWorkflowTaskResponse> {
3766        let poll_request_id = unique_request_id("rust-workflow-poll");
3767        self.poll_workflow_task_response_with_request_id(
3768            worker_id,
3769            task_queue,
3770            timeout,
3771            &poll_request_id,
3772            1,
3773        )
3774        .await
3775    }
3776
3777    async fn poll_workflow_task_response_with_request_id(
3778        &self,
3779        worker_id: &str,
3780        task_queue: &str,
3781        timeout: Duration,
3782        poll_request_id: &str,
3783        transport_retries: usize,
3784    ) -> Result<PollWorkflowTaskResponse> {
3785        let body = json!({
3786            "worker_id": worker_id,
3787            "task_queue": task_queue,
3788            "poll_request_id": poll_request_id,
3789            "timeout_seconds": long_poll_timeout_seconds(timeout),
3790        });
3791        let mut data: PollWorkflowTaskResponse = self
3792            .poll_request_json(
3793                "/worker/workflow-tasks/poll",
3794                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3795                &body,
3796                timeout + Duration::from_secs(5),
3797                transport_retries,
3798            )
3799            .await?;
3800
3801        if let Some(task) = data.task.as_mut() {
3802            self.fetch_remaining_workflow_history(worker_id, task)
3803                .await?;
3804        }
3805
3806        Ok(data)
3807    }
3808
3809    async fn fetch_remaining_workflow_history(
3810        &self,
3811        worker_id: &str,
3812        task: &mut WorkflowTask,
3813    ) -> Result<()> {
3814        let mut next_token = task.next_history_page_token.clone();
3815
3816        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3817            let lease_owner = task
3818                .lease_owner
3819                .clone()
3820                .unwrap_or_else(|| worker_id.to_string());
3821            let page = self
3822                .workflow_task_history_page(
3823                    &task.task_id,
3824                    &lease_owner,
3825                    task.workflow_task_attempt,
3826                    &token,
3827                )
3828                .await?;
3829
3830            task.append_history_page(page);
3831
3832            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3833                return Err(Error::Codec(
3834                    "workflow history pagination returned the same page token".to_string(),
3835                ));
3836            }
3837
3838            next_token = task.next_history_page_token.clone();
3839        }
3840
3841        Ok(())
3842    }
3843
3844    async fn workflow_task_history_page(
3845        &self,
3846        task_id: &str,
3847        lease_owner: &str,
3848        workflow_task_attempt: u64,
3849        next_history_page_token: &str,
3850    ) -> Result<WorkflowTaskHistoryPage> {
3851        let body = json!({
3852            "lease_owner": lease_owner,
3853            "workflow_task_attempt": workflow_task_attempt,
3854            "next_history_page_token": next_history_page_token
3855        });
3856        let path = format!("/worker/workflow-tasks/{task_id}/history");
3857
3858        self.request_json(
3859            reqwest::Method::POST,
3860            &path,
3861            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3862            Some(&body),
3863        )
3864        .await
3865    }
3866
3867    pub async fn complete_workflow_task(
3868        &self,
3869        task_id: &str,
3870        lease_owner: &str,
3871        workflow_task_attempt: u64,
3872        commands: Vec<Value>,
3873    ) -> Result<Value> {
3874        self.complete_workflow_task_with_message_streams(
3875            task_id,
3876            lease_owner,
3877            workflow_task_attempt,
3878            commands,
3879            Vec::new(),
3880            Vec::new(),
3881        )
3882        .await
3883    }
3884
3885    async fn complete_workflow_task_with_message_streams(
3886        &self,
3887        task_id: &str,
3888        lease_owner: &str,
3889        workflow_task_attempt: u64,
3890        commands: Vec<Value>,
3891        message_stream_cursors: Vec<Value>,
3892        message_stream_waits: Vec<Value>,
3893    ) -> Result<Value> {
3894        validate_workflow_task_commands(&commands)?;
3895        let has_message_stream_metadata =
3896            !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3897        if has_message_stream_metadata
3898            && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3899        {
3900            return Err(Error::Codec(
3901                "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3902                    .to_string(),
3903            ));
3904        }
3905        let protocol_version = workflow_completion_protocol_version_with_message_streams(
3906            &commands,
3907            has_message_stream_metadata,
3908        );
3909        let mut body = json!({
3910            "lease_owner": lease_owner,
3911            "workflow_task_attempt": workflow_task_attempt,
3912            "commands": commands
3913        });
3914        if !message_stream_cursors.is_empty() {
3915            body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3916        }
3917        if !message_stream_waits.is_empty() {
3918            body["message_stream_waits"] = Value::Array(message_stream_waits);
3919        }
3920        let path = format!("/worker/workflow-tasks/{task_id}/complete");
3921        self.request_json(
3922            reqwest::Method::POST,
3923            &path,
3924            RequestProtocol::Worker(protocol_version),
3925            Some(&body),
3926        )
3927        .await
3928    }
3929
3930    pub async fn fail_workflow_task(
3931        &self,
3932        task_id: &str,
3933        lease_owner: &str,
3934        workflow_task_attempt: u64,
3935        message: impl Into<String>,
3936    ) -> Result<Value> {
3937        self.fail_workflow_task_with_type(
3938            task_id,
3939            lease_owner,
3940            workflow_task_attempt,
3941            message,
3942            "RustWorkflowTaskFailure",
3943        )
3944        .await
3945    }
3946
3947    async fn fail_workflow_task_with_type(
3948        &self,
3949        task_id: &str,
3950        lease_owner: &str,
3951        workflow_task_attempt: u64,
3952        message: impl Into<String>,
3953        failure_type: &str,
3954    ) -> Result<Value> {
3955        let body = json!({
3956            "lease_owner": lease_owner,
3957            "workflow_task_attempt": workflow_task_attempt,
3958            "failure": {
3959                "message": message.into(),
3960                "type": failure_type
3961            }
3962        });
3963        let path = format!("/worker/workflow-tasks/{task_id}/fail");
3964        self.request_json(
3965            reqwest::Method::POST,
3966            &path,
3967            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3968            Some(&body),
3969        )
3970        .await
3971    }
3972
3973    pub async fn poll_activity_task(
3974        &self,
3975        worker_id: &str,
3976        task_queue: &str,
3977        timeout: Duration,
3978    ) -> Result<Option<ActivityTask>> {
3979        Ok(self
3980            .poll_activity_task_response(worker_id, task_queue, timeout)
3981            .await?
3982            .task)
3983    }
3984
3985    /// Poll an activity task while preserving server stop and drain metadata.
3986    pub async fn poll_activity_task_response(
3987        &self,
3988        worker_id: &str,
3989        task_queue: &str,
3990        timeout: Duration,
3991    ) -> Result<PollActivityTaskResponse> {
3992        let poll_request_id = unique_request_id("rust-activity-poll");
3993        self.poll_activity_task_response_with_request_id(
3994            worker_id,
3995            task_queue,
3996            timeout,
3997            &poll_request_id,
3998            1,
3999        )
4000        .await
4001    }
4002
4003    async fn poll_activity_task_response_with_request_id(
4004        &self,
4005        worker_id: &str,
4006        task_queue: &str,
4007        timeout: Duration,
4008        poll_request_id: &str,
4009        transport_retries: usize,
4010    ) -> Result<PollActivityTaskResponse> {
4011        let body = json!({
4012            "worker_id": worker_id,
4013            "task_queue": task_queue,
4014            "poll_request_id": poll_request_id,
4015            "timeout_seconds": long_poll_timeout_seconds(timeout),
4016        });
4017        let data: PollActivityTaskResponse = self
4018            .poll_request_json(
4019                "/worker/activity-tasks/poll",
4020                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4021                &body,
4022                timeout + Duration::from_secs(5),
4023                transport_retries,
4024            )
4025            .await?;
4026        Ok(data)
4027    }
4028
4029    pub async fn complete_activity_task<T: Serialize>(
4030        &self,
4031        task_id: &str,
4032        activity_attempt_id: &str,
4033        lease_owner: &str,
4034        result: T,
4035        codec: &str,
4036    ) -> Result<Value> {
4037        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
4038        let body = json!({
4039            "activity_attempt_id": activity_attempt_id,
4040            "lease_owner": lease_owner,
4041            "result": result
4042        });
4043        let path = format!("/worker/activity-tasks/{task_id}/complete");
4044        activity_task_response(
4045            self.request_json(
4046                reqwest::Method::POST,
4047                &path,
4048                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4049                Some(&body),
4050            )
4051            .await,
4052            "complete",
4053            task_id,
4054            activity_attempt_id,
4055        )
4056    }
4057
4058    pub async fn fail_activity_task(
4059        &self,
4060        task_id: &str,
4061        activity_attempt_id: &str,
4062        lease_owner: &str,
4063        message: impl Into<String>,
4064        non_retryable: bool,
4065    ) -> Result<Value> {
4066        let body = json!({
4067            "activity_attempt_id": activity_attempt_id,
4068            "lease_owner": lease_owner,
4069            "failure": {
4070                "message": message.into(),
4071                "type": "RustActivityFailure",
4072                "non_retryable": non_retryable
4073            }
4074        });
4075        let path = format!("/worker/activity-tasks/{task_id}/fail");
4076        activity_task_response(
4077            self.request_json(
4078                reqwest::Method::POST,
4079                &path,
4080                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4081                Some(&body),
4082            )
4083            .await,
4084            "fail",
4085            task_id,
4086            activity_attempt_id,
4087        )
4088    }
4089
4090    pub async fn heartbeat_activity_task<T: Serialize>(
4091        &self,
4092        task_id: &str,
4093        activity_attempt_id: &str,
4094        lease_owner: &str,
4095        details: T,
4096    ) -> Result<ActivityHeartbeatResponse> {
4097        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
4098        let body = json!({
4099            "activity_attempt_id": activity_attempt_id,
4100            "lease_owner": lease_owner,
4101            "details": details
4102        });
4103        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4104        activity_task_response(
4105            self.request_json(
4106                reqwest::Method::POST,
4107                &path,
4108                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4109                Some(&body),
4110            )
4111            .await,
4112            "heartbeat",
4113            task_id,
4114            activity_attempt_id,
4115        )
4116    }
4117
4118    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4119        &self,
4120        method: reqwest::Method,
4121        path: &str,
4122        protocol: RequestProtocol,
4123        body: Option<&B>,
4124    ) -> Result<T> {
4125        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4126            .await
4127    }
4128
4129    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4130        &self,
4131        method: reqwest::Method,
4132        path: &str,
4133        protocol: RequestProtocol,
4134        body: Option<&B>,
4135        timeout: Duration,
4136    ) -> Result<T> {
4137        let auth_token = self.auth_token(protocol)?;
4138        let mut request = self
4139            .http
4140            .request(method.clone(), format!("{}/api{}", self.base_url, path))
4141            .timeout(timeout)
4142            .header(reqwest::header::ACCEPT, "application/json")
4143            .header(reqwest::header::CONTENT_TYPE, "application/json")
4144            .header("X-Namespace", &self.namespace);
4145
4146        match protocol {
4147            RequestProtocol::Worker(version) => {
4148                request = request.header("X-Durable-Workflow-Protocol-Version", version);
4149            }
4150            RequestProtocol::ControlPlane => {
4151                request = request.header(
4152                    "X-Durable-Workflow-Control-Plane-Version",
4153                    CONTROL_PLANE_VERSION,
4154                );
4155            }
4156        }
4157
4158        if let Some(token) = auth_token {
4159            request = request.bearer_auth(token);
4160        }
4161
4162        if let Some(body) = body {
4163            let mut body = serde_json::to_value(body)?;
4164            if matches!(
4165                method,
4166                reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH
4167            ) {
4168                self.externalize_runtime_payloads(&mut body, path, protocol)
4169                    .await?;
4170            }
4171            request = request.json(&body);
4172        }
4173
4174        let request = request.build()?;
4175        let request_body = request
4176            .body()
4177            .and_then(reqwest::Body::as_bytes)
4178            .and_then(|body| serde_json::from_slice::<Value>(body).ok());
4179        let poll_request_id = path.ends_with("/poll").then(|| {
4180            request_body
4181                .as_ref()
4182                .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4183                .unwrap_or_default()
4184        });
4185        let mut storage_retries = 0_usize;
4186
4187        loop {
4188            // Keep the serialized body and lease/poll identity across admission pauses.
4189            let response = self
4190                .http
4191                .execute(request.try_clone().ok_or_else(|| {
4192                    Error::WorkerLoop("worker request body cannot be retried".to_string())
4193                })?)
4194                .await?;
4195            let status = response.status();
4196            let bytes = response.bytes().await?;
4197
4198            if !status.is_success() {
4199                let body = String::from_utf8_lossy(&bytes).to_string();
4200                if let Some(protocol) = protocol_failure(status, &body) {
4201                    return Err(Error::Protocol(protocol));
4202                }
4203                let error = Error::Http { status, body };
4204                if self
4205                    .wait_for_storage_admission(
4206                        &error,
4207                        protocol,
4208                        Some(path),
4209                        poll_request_id.as_deref(),
4210                        request_body.as_ref(),
4211                        &mut storage_retries,
4212                    )
4213                    .await
4214                {
4215                    continue;
4216                }
4217                return Err(error);
4218            }
4219
4220            if bytes.is_empty() {
4221                return Ok(serde_json::from_value(Value::Null)?);
4222            }
4223
4224            let mut value: Value = serde_json::from_slice(&bytes)?;
4225            self.resolve_runtime_payloads(&mut value, path, protocol)
4226                .await?;
4227            return Ok(serde_json::from_value(value)?);
4228        }
4229    }
4230
4231    async fn wait_for_storage_admission(
4232        &self,
4233        error: &Error,
4234        protocol: RequestProtocol,
4235        path: Option<&str>,
4236        poll_request_id: Option<&str>,
4237        request_body: Option<&Value>,
4238        retries: &mut usize,
4239    ) -> bool {
4240        let Some(admission) = self
4241            .worker_storage_admission
4242            .as_ref()
4243            .filter(|_| matches!(protocol, RequestProtocol::Worker(_)))
4244        else {
4245            return false;
4246        };
4247        let Some(advertised_delay) = worker_storage_admission_retry_after(error, poll_request_id)
4248            .or_else(|| {
4249                path.and_then(|path| {
4250                    worker_backend_unavailable_retry_after(error, path, poll_request_id)
4251                        .or_else(|| worker_fenced_completion_retry_after(error, path, request_body))
4252                })
4253            })
4254        else {
4255            return false;
4256        };
4257        *retries = retries.saturating_add(1);
4258        let delay = worker_retry_delay(admission.policy, *retries)
4259            .max(advertised_delay)
4260            .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4261        let deadline = tokio::time::Instant::now() + delay;
4262        loop {
4263            if admission.stop.load(Ordering::SeqCst) {
4264                return false;
4265            }
4266            let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
4267            if remaining.is_zero() {
4268                return true;
4269            }
4270            tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4271        }
4272    }
4273
4274    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4275        &self,
4276        path: &str,
4277        protocol: RequestProtocol,
4278        body: &B,
4279        timeout: Duration,
4280        max_retries: usize,
4281    ) -> Result<T> {
4282        let mut retries = 0;
4283
4284        loop {
4285            let response = self
4286                .request_json_with_timeout(
4287                    reqwest::Method::POST,
4288                    path,
4289                    protocol,
4290                    Some(body),
4291                    timeout,
4292                )
4293                .await;
4294
4295            match response {
4296                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4297                response => return worker_poll_response(response),
4298            }
4299        }
4300    }
4301
4302    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4303        match protocol {
4304            RequestProtocol::Worker(_) => {
4305                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4306                    return Ok(Some(token));
4307                }
4308                if self.control_token.is_some() {
4309                    return Err(Error::MissingRoleCredentials {
4310                        role: "worker",
4311                        opposite_role: "control",
4312                    });
4313                }
4314                Ok(None)
4315            }
4316            RequestProtocol::ControlPlane => {
4317                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4318                    return Ok(Some(token));
4319                }
4320                if self.worker_token.is_some() {
4321                    return Err(Error::MissingRoleCredentials {
4322                        role: "control",
4323                        opposite_role: "worker",
4324                    });
4325                }
4326                Ok(None)
4327            }
4328        }
4329    }
4330}
4331
4332fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4333    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4334    let reason = body
4335        .get("reason")
4336        .and_then(Value::as_str)
4337        .unwrap_or("query_rejected")
4338        .to_string();
4339    let message = body
4340        .get("message")
4341        .or_else(|| body.get("error"))
4342        .and_then(Value::as_str)
4343        .unwrap_or("workflow query was rejected")
4344        .to_string();
4345
4346    QueryFailure {
4347        status: status.as_u16(),
4348        reason,
4349        message,
4350        body,
4351    }
4352}
4353
4354fn workflow_command_result(
4355    command: WorkflowCommandKind,
4356    data: Value,
4357    workflow_id: &str,
4358    run_id: Option<&str>,
4359) -> WorkflowCommandResult {
4360    WorkflowCommandResult {
4361        command,
4362        workflow_id: data
4363            .get("workflow_id")
4364            .and_then(Value::as_str)
4365            .unwrap_or(workflow_id)
4366            .to_string(),
4367        run_id: data
4368            .get("run_id")
4369            .and_then(Value::as_str)
4370            .or(run_id)
4371            .map(str::to_string),
4372        outcome: data
4373            .get("outcome")
4374            .and_then(Value::as_str)
4375            .map(str::to_string),
4376        reason: data
4377            .get("reason")
4378            .and_then(Value::as_str)
4379            .map(str::to_string),
4380        command_status: data
4381            .get("command_status")
4382            .and_then(Value::as_str)
4383            .map(str::to_string),
4384        raw: data,
4385    }
4386}
4387
4388fn workflow_command_rejection(
4389    command: WorkflowCommandKind,
4390    status: reqwest::StatusCode,
4391    raw_body: String,
4392    workflow_id: &str,
4393    run_id: Option<&str>,
4394) -> WorkflowCommandRejection {
4395    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4396    WorkflowCommandRejection {
4397        command,
4398        status: status.as_u16(),
4399        reason: body
4400            .get("reason")
4401            .and_then(Value::as_str)
4402            .unwrap_or("workflow_command_rejected")
4403            .to_string(),
4404        message: body
4405            .get("message")
4406            .or_else(|| body.get("error"))
4407            .and_then(Value::as_str)
4408            .unwrap_or("workflow lifecycle command was rejected")
4409            .to_string(),
4410        workflow_id: body
4411            .get("workflow_id")
4412            .and_then(Value::as_str)
4413            .unwrap_or(workflow_id)
4414            .to_string(),
4415        run_id: body
4416            .get("run_id")
4417            .and_then(Value::as_str)
4418            .or(run_id)
4419            .map(str::to_string),
4420        target_scope: body
4421            .get("target_scope")
4422            .and_then(Value::as_str)
4423            .map(str::to_string),
4424        body,
4425    }
4426}
4427
4428fn query_task_response(response: Result<Value>) -> Result<Value> {
4429    match response {
4430        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4431        response => response,
4432    }
4433}
4434
4435fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4436    match response {
4437        Err(Error::Http { status, body })
4438            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4439        {
4440            Ok(serde_json::from_str(&body)?)
4441        }
4442        response => response,
4443    }
4444}
4445
4446fn worker_poll_body_is_stop(body: &str) -> bool {
4447    serde_json::from_str::<Value>(body)
4448        .ok()
4449        .is_some_and(|body| {
4450            worker_poll_is_stop(
4451                body.get("poll_status").and_then(Value::as_str),
4452                body.get("reason").and_then(Value::as_str),
4453            )
4454        })
4455}
4456
4457fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4458    matches!(poll_status, Some("draining" | "stopped"))
4459        || matches!(reason, Some("worker_draining" | "worker_stopped"))
4460}
4461
4462fn query_task_rejection_is_final(error: &Error) -> bool {
4463    matches!(
4464        error,
4465        Error::QueryFailed(failure)
4466            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4467    )
4468}
4469
4470fn activity_task_response<T>(
4471    response: Result<T>,
4472    operation: &str,
4473    task_id: &str,
4474    activity_attempt_id: &str,
4475) -> Result<T> {
4476    match response {
4477        Err(Error::Http { status, body }) => {
4478            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4479            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4480                operation: operation.to_string(),
4481                status: status.as_u16(),
4482                reason: body
4483                    .get("reason")
4484                    .and_then(Value::as_str)
4485                    .unwrap_or("activity_task_rejected")
4486                    .to_string(),
4487                task_id: body
4488                    .get("task_id")
4489                    .and_then(Value::as_str)
4490                    .unwrap_or(task_id)
4491                    .to_string(),
4492                activity_attempt_id: body
4493                    .get("activity_attempt_id")
4494                    .and_then(Value::as_str)
4495                    .unwrap_or(activity_attempt_id)
4496                    .to_string(),
4497                cancel_requested: body
4498                    .get("cancel_requested")
4499                    .and_then(Value::as_bool)
4500                    .unwrap_or(false),
4501                can_continue: body.get("can_continue").and_then(Value::as_bool),
4502                run_closed_reason: body
4503                    .get("run_closed_reason")
4504                    .and_then(Value::as_str)
4505                    .map(str::to_string),
4506                body,
4507            }))
4508        }
4509        response => response,
4510    }
4511}
4512
4513fn activity_task_rejection_is_final(error: &Error) -> bool {
4514    matches!(
4515        error,
4516        Error::ActivityTaskRejected(rejection)
4517            if matches!(
4518                rejection.reason.as_str(),
4519                "run_cancelled"
4520                    | "run_terminated"
4521                    | "attempt_closed"
4522                    | "stale_attempt"
4523                    | "activity_cancelled"
4524                    | "task_cancelled"
4525                    | "run_closed"
4526                    | "activity_not_running"
4527                    | "attempt_not_found"
4528            )
4529    )
4530}
4531
4532fn workflow_task_completion_is_terminal_timeout(
4533    error: &Error,
4534    task_id: &str,
4535    workflow_task_attempt: u64,
4536    run_id: Option<&str>,
4537) -> bool {
4538    let Error::Http { status, body } = error else {
4539        return false;
4540    };
4541    if *status != reqwest::StatusCode::CONFLICT {
4542        return false;
4543    }
4544
4545    let Some(run_id) = run_id else {
4546        return false;
4547    };
4548    let Ok(body) = serde_json::from_str::<Value>(body) else {
4549        return false;
4550    };
4551
4552    body.get("recorded").and_then(Value::as_bool) == Some(false)
4553        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4554        && body.get("run_status").and_then(Value::as_str) == Some("failed")
4555        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4556        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4557        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4558}
4559
4560fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4561    let body: Value = serde_json::from_str(raw_body).ok()?;
4562    let reason = body.get("reason")?.as_str()?;
4563    if !matches!(
4564        reason,
4565        "missing_protocol_version"
4566            | "unsupported_protocol_version"
4567            | "missing_control_plane_version"
4568            | "unsupported_control_plane_version"
4569    ) {
4570        return None;
4571    }
4572
4573    Some(ProtocolFailure {
4574        status: status.as_u16(),
4575        reason: reason.to_string(),
4576        message: body
4577            .get("message")
4578            .or_else(|| body.get("error"))
4579            .and_then(Value::as_str)
4580            .unwrap_or("protocol version rejected")
4581            .to_string(),
4582        supported_version: body
4583            .get("supported_version")
4584            .and_then(Value::as_str)
4585            .map(str::to_string),
4586        requested_version: body
4587            .get("requested_version")
4588            .and_then(Value::as_str)
4589            .map(str::to_string),
4590        body,
4591    })
4592}
4593
4594fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4595    timeout
4596        .as_secs()
4597        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4598        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4599}
4600
4601fn worker_operation_is_retryable(error: &Error) -> bool {
4602    if worker_poll_capacity_retry_after(error).is_some()
4603        || worker_storage_admission_body(error).is_some()
4604        || worker_backend_unavailable_body(error).is_some()
4605        || worker_operation_is_explicitly_non_retryable(error)
4606    {
4607        return false;
4608    }
4609
4610    match error {
4611        Error::Transport(error) => {
4612            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4613        }
4614        Error::Http { status, .. } => {
4615            matches!(
4616                *status,
4617                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4618            ) || status.is_server_error()
4619        }
4620        _ => false,
4621    }
4622}
4623
4624fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4625    let body: Value = match error {
4626        Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4627        Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4628        _ => return None,
4629    };
4630    matches!(
4631        body.get("reason").and_then(Value::as_str),
4632        Some("storage_pressure" | "storage_admission_unavailable")
4633    )
4634    .then_some(body)
4635}
4636
4637fn worker_storage_admission_retry_after(
4638    error: &Error,
4639    poll_request_id: Option<&str>,
4640) -> Option<Duration> {
4641    let Error::Http { status, .. } = error else {
4642        return None;
4643    };
4644    let body = worker_storage_admission_body(error)?;
4645    let delay = body.get("retry_after_seconds")?.as_u64()?;
4646    if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4647        || delay == 0
4648        || body.get("retryable") != Some(&Value::Bool(true))
4649        || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4650        || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4651        || body
4652            .get("request_admitted")
4653            .is_some_and(|admitted| admitted != &Value::Bool(false))
4654    {
4655        return None;
4656    }
4657    match poll_request_id {
4658        Some(id) => {
4659            if id.is_empty()
4660                || body.get("task") != Some(&Value::Null)
4661                || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4662                || body.get("poll_status") != body.get("reason")
4663                || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4664                || body.get("claim_admitted") != Some(&Value::Bool(false))
4665            {
4666                return None;
4667            }
4668        }
4669        None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4670        None => {}
4671    }
4672    Some(Duration::from_secs(delay))
4673}
4674
4675fn worker_backend_unavailable_retry_after(
4676    error: &Error,
4677    path: &str,
4678    poll_request_id: Option<&str>,
4679) -> Option<Duration> {
4680    let operation = match path {
4681        "/worker/register" => "register_worker",
4682        "/worker/heartbeat" => "heartbeat_worker",
4683        "/worker/workflow-tasks/poll" => "poll_workflow_task",
4684        "/worker/activity-tasks/poll" => "poll_activity_task",
4685        "/worker/query-tasks/poll" => "poll_query_task",
4686        "/worker/update-validation-tasks/poll" => "poll_update_validation_task",
4687        _ => return None,
4688    };
4689    let body = worker_backend_unavailable_body(error)?;
4690    let delay = body.get("retry_after_seconds")?.as_u64()?;
4691    if delay == 0
4692        || body.get("operation")?.as_str()? != operation
4693        || body.get("outcome")?.as_str()? != "unknown"
4694        || body.get("retryable") != Some(&Value::Bool(true))
4695        || body.get("worker_id")?.as_str()?.is_empty()
4696    {
4697        return None;
4698    }
4699    if let Some(id) = poll_request_id {
4700        if id.is_empty()
4701            || body.get("task") != Some(&Value::Null)
4702            || body.get("poll_status")?.as_str()? != "backend_unavailable"
4703            || body.get("poll_request_id")?.as_str()? != id
4704            || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4705        {
4706            return None;
4707        }
4708    } else if path.ends_with("/poll") {
4709        return None;
4710    }
4711    Some(Duration::from_secs(delay))
4712}
4713
4714fn worker_fenced_completion_retry_after(
4715    error: &Error,
4716    path: &str,
4717    request_body: Option<&Value>,
4718) -> Option<Duration> {
4719    let (operation, task_id, attempt_field) = if let Some(task_id) = path
4720        .strip_prefix("/worker/workflow-tasks/")
4721        .and_then(|tail| tail.strip_suffix("/complete"))
4722    {
4723        ("complete_workflow_task", task_id, "workflow_task_attempt")
4724    } else if let Some(task_id) = path
4725        .strip_prefix("/worker/activity-tasks/")
4726        .and_then(|tail| tail.strip_suffix("/complete"))
4727    {
4728        ("complete_activity_task", task_id, "activity_attempt_id")
4729    } else {
4730        return None;
4731    };
4732    if task_id.is_empty() || task_id.contains('/') {
4733        return None;
4734    }
4735
4736    let request = request_body?.as_object()?;
4737    let lease_owner = request.get("lease_owner")?.as_str()?;
4738    if lease_owner.is_empty() {
4739        return None;
4740    }
4741    let attempt = request.get(attempt_field)?;
4742    if (attempt_field == "workflow_task_attempt" && attempt.as_u64().filter(|id| *id > 0).is_none())
4743        || (attempt_field == "activity_attempt_id"
4744            && attempt.as_str().filter(|id| !id.is_empty()).is_none())
4745    {
4746        return None;
4747    }
4748
4749    let body = worker_backend_unavailable_body(error)?;
4750    let delay = body.get("retry_after_seconds")?.as_u64()?;
4751    if delay == 0
4752        || body.get("operation")?.as_str()? != operation
4753        || body.get("outcome")?.as_str()? != "unknown"
4754        || body.get("retryable") != Some(&Value::Bool(true))
4755        || body.get("worker_id")?.as_str()? != lease_owner
4756        || body.get("task_queue") != Some(&Value::Null)
4757        || body.get("task_id")?.as_str()? != task_id
4758        || body.get("lease_owner")?.as_str()? != lease_owner
4759        || body.get(attempt_field) != Some(attempt)
4760    {
4761        return None;
4762    }
4763
4764    Some(Duration::from_secs(delay))
4765}
4766
4767fn worker_backend_unavailable_body(error: &Error) -> Option<Value> {
4768    let Error::Http { status, body } = error else {
4769        return None;
4770    };
4771    if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE {
4772        return None;
4773    }
4774    let body: Value = serde_json::from_str(body).ok()?;
4775    (body.get("reason")?.as_str()? == "backend_unavailable").then_some(body)
4776}
4777
4778fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4779    let Error::Http { body, .. } = error else {
4780        return false;
4781    };
4782
4783    serde_json::from_str::<Value>(body)
4784        .ok()
4785        .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4786        == Some(false)
4787}
4788
4789fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4790    let Error::Http { status, body } = error else {
4791        return None;
4792    };
4793    if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4794        return None;
4795    }
4796
4797    let body = serde_json::from_str::<Value>(body).ok()?;
4798    let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4799        == Some("long_poll_capacity_exhausted")
4800        || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4801    if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4802        return None;
4803    }
4804
4805    Some(Duration::from_secs(
4806        body.get("retry_after_seconds")
4807            .and_then(Value::as_u64)
4808            .unwrap_or_default(),
4809    ))
4810}
4811
4812fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4813    let exponent = retry.saturating_sub(1).min(31) as u32;
4814    policy
4815        .initial_backoff
4816        .saturating_mul(1_u32 << exponent)
4817        .min(policy.max_backoff)
4818}
4819
4820#[derive(Debug)]
4821pub struct ClientBuilder {
4822    base_url: String,
4823    token: Option<String>,
4824    control_token: Option<String>,
4825    worker_token: Option<String>,
4826    namespace: String,
4827    timeout: Duration,
4828    max_external_payload_bytes: usize,
4829}
4830
4831impl ClientBuilder {
4832    pub fn token(mut self, token: Option<String>) -> Self {
4833        self.token = token;
4834        self
4835    }
4836
4837    pub fn control_token(mut self, token: Option<String>) -> Self {
4838        self.control_token = token;
4839        self
4840    }
4841
4842    pub fn worker_token(mut self, token: Option<String>) -> Self {
4843        self.worker_token = token;
4844        self
4845    }
4846
4847    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4848        self.namespace = namespace.into();
4849        self
4850    }
4851
4852    pub fn timeout(mut self, timeout: Duration) -> Self {
4853        self.timeout = timeout;
4854        self
4855    }
4856
4857    /// Maximum unique external-payload bytes fetched per response (default 64 MiB).
4858    /// References are fetched only from this client's authenticated runtime.
4859    pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
4860        self.max_external_payload_bytes = bytes;
4861        self
4862    }
4863
4864    pub fn build(self) -> Result<Client> {
4865        let base_url = self.base_url.trim_end_matches('/').to_string();
4866        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4867            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4868            .unwrap_or_else(|_| base_url.ends_with("/api"));
4869
4870        if has_sdk_api_suffix {
4871            return Err(Error::InvalidBaseUrl);
4872        }
4873
4874        Ok(Client {
4875            http: reqwest::Client::builder()
4876                .timeout(self.timeout)
4877                .redirect(reqwest::redirect::Policy::none())
4878                .build()?,
4879            base_url,
4880            token: self.token,
4881            control_token: self.control_token,
4882            worker_token: self.worker_token,
4883            namespace: self.namespace,
4884            max_external_payload_bytes: self.max_external_payload_bytes,
4885            worker_storage_admission: None,
4886            runtime_upload_policy: Arc::new(Mutex::new([None, None])),
4887        })
4888    }
4889}
4890
4891#[derive(Clone, Debug)]
4892pub struct WorkflowHandle {
4893    client: Client,
4894    pub workflow_id: String,
4895    pub run_id: Option<String>,
4896    pub workflow_type: String,
4897}
4898
4899impl WorkflowHandle {
4900    /// Describe whichever run is current for this stable workflow instance.
4901    pub async fn describe(&self) -> Result<WorkflowDescription> {
4902        self.client.describe_workflow(&self.workflow_id).await
4903    }
4904
4905    /// Describe the run identity originally selected by this handle.
4906    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4907        let run_id = self.run_id.as_deref().ok_or_else(|| {
4908            Error::Codec("run_id is required for selected-run description".to_string())
4909        })?;
4910        self.client
4911            .describe_workflow_run(&self.workflow_id, run_id)
4912            .await
4913    }
4914
4915    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4916        self.client
4917            .signal_workflow(&self.workflow_id, signal_name, input)
4918            .await
4919    }
4920
4921    pub async fn append_message<T: Serialize>(
4922        &self,
4923        stream_name: &str,
4924        message_id: &str,
4925        input: T,
4926    ) -> Result<Value> {
4927        self.client
4928            .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4929            .await
4930    }
4931
4932    /// Signal only if this handle's selected run is still current.
4933    pub async fn signal_selected_run<T: Serialize>(
4934        &self,
4935        signal_name: &str,
4936        input: T,
4937    ) -> Result<Value> {
4938        let run_id = self.run_id.as_deref().ok_or_else(|| {
4939            Error::Codec("run_id is required for selected-run signaling".to_string())
4940        })?;
4941        self.client
4942            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4943            .await
4944    }
4945
4946    /// Close whichever run is current as cancelled, without workflow cleanup.
4947    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4948        self.client
4949            .cancel_workflow(&self.workflow_id, options)
4950            .await
4951    }
4952
4953    /// Request cancellation only if this handle's selected run is still current.
4954    pub async fn cancel_selected_run(
4955        &self,
4956        options: WorkflowCommandOptions,
4957    ) -> Result<WorkflowCommandResult> {
4958        let run_id = self.run_id.as_deref().ok_or_else(|| {
4959            Error::Codec("run_id is required for selected-run cancellation".to_string())
4960        })?;
4961        self.client
4962            .cancel_workflow_run(&self.workflow_id, run_id, options)
4963            .await
4964    }
4965
4966    /// Forcefully terminate whichever run is current.
4967    pub async fn terminate(
4968        &self,
4969        options: WorkflowCommandOptions,
4970    ) -> Result<WorkflowCommandResult> {
4971        self.client
4972            .terminate_workflow(&self.workflow_id, options)
4973            .await
4974    }
4975
4976    /// Terminate only if this handle's selected run is still current.
4977    pub async fn terminate_selected_run(
4978        &self,
4979        options: WorkflowCommandOptions,
4980    ) -> Result<WorkflowCommandResult> {
4981        let run_id = self.run_id.as_deref().ok_or_else(|| {
4982            Error::Codec("run_id is required for selected-run termination".to_string())
4983        })?;
4984        self.client
4985            .terminate_workflow_run(&self.workflow_id, run_id, options)
4986            .await
4987    }
4988
4989    /// Continue this handle's failed run without repeating completed activities.
4990    pub async fn redrive(&self, request_id: Option<&str>) -> Result<WorkflowRedriveResult> {
4991        let run_id = self
4992            .run_id
4993            .as_deref()
4994            .ok_or_else(|| Error::Codec("run_id is required for redrive".to_string()))?;
4995        self.client
4996            .redrive_workflow_run(&self.workflow_id, run_id, request_id)
4997            .await
4998    }
4999
5000    /// Execute a named, read-only query against this workflow.
5001    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
5002        self.client
5003            .query_workflow(&self.workflow_id, query_name, input)
5004            .await
5005    }
5006
5007    pub async fn query_avro_value<T: Serialize>(
5008        &self,
5009        query_name: &str,
5010        input: T,
5011    ) -> Result<AvroValue> {
5012        self.client
5013            .query_workflow_avro_value(&self.workflow_id, query_name, input)
5014            .await
5015    }
5016
5017    pub async fn update<T: Serialize>(
5018        &self,
5019        update_name: &str,
5020        input: T,
5021        request_id: Option<&str>,
5022    ) -> Result<Value> {
5023        self.client
5024            .update_workflow(&self.workflow_id, update_name, input, request_id)
5025            .await
5026    }
5027
5028    pub async fn update_avro_value<T: Serialize>(
5029        &self,
5030        update_name: &str,
5031        input: T,
5032        request_id: Option<&str>,
5033    ) -> Result<AvroValue> {
5034        self.client
5035            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
5036            .await
5037    }
5038
5039    /// Query only if this handle's selected run is still current.
5040    pub async fn query_selected_run<T: Serialize>(
5041        &self,
5042        query_name: &str,
5043        input: T,
5044    ) -> Result<Value> {
5045        let run_id = self
5046            .run_id
5047            .as_deref()
5048            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
5049        self.client
5050            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
5051            .await
5052    }
5053
5054    /// Await the final terminal outcome of the current continue-as-new chain.
5055    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
5056        self.result_target(options, None).await
5057    }
5058
5059    /// Await the final result without projecting Avro bytes through JSON.
5060    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
5061        self.result_avro_value_target(options, None).await
5062    }
5063
5064    /// Await the final result and decode it into a Serde application type.
5065    pub async fn result_typed<T: DeserializeOwned>(
5066        &self,
5067        options: WorkflowResultOptions,
5068    ) -> Result<T> {
5069        let result = self.result_avro_value(options).await?;
5070        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5071    }
5072
5073    /// Await only the run identity originally selected by this handle.
5074    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
5075        let run_id = self.run_id.as_deref().ok_or_else(|| {
5076            Error::Codec("run_id is required for selected-run result".to_string())
5077        })?;
5078        self.result_target(options, Some(run_id)).await
5079    }
5080
5081    /// Await the selected run's result on the lossless Avro Value surface.
5082    pub async fn result_selected_run_avro_value(
5083        &self,
5084        options: WorkflowResultOptions,
5085    ) -> Result<AvroValue> {
5086        let run_id = self.run_id.as_deref().ok_or_else(|| {
5087            Error::Codec("run_id is required for selected-run result".to_string())
5088        })?;
5089        self.result_avro_value_target(options, Some(run_id)).await
5090    }
5091
5092    /// Await the selected run and decode its result into a Serde type.
5093    pub async fn result_selected_run_typed<T: DeserializeOwned>(
5094        &self,
5095        options: WorkflowResultOptions,
5096    ) -> Result<T> {
5097        let result = self.result_selected_run_avro_value(options).await?;
5098        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5099    }
5100
5101    async fn result_avro_value_target(
5102        &self,
5103        options: WorkflowResultOptions,
5104        selected_run_id: Option<&str>,
5105    ) -> Result<AvroValue> {
5106        let started = Instant::now();
5107
5108        loop {
5109            let description = match selected_run_id {
5110                Some(run_id) => {
5111                    self.client
5112                        .describe_workflow_run(&self.workflow_id, run_id)
5113                        .await?
5114                }
5115                None => self.describe().await?,
5116            };
5117            if description.is_completed() {
5118                return description.output_avro_value.ok_or_else(|| {
5119                    Error::Codec(
5120                        "missing_payload_envelope: typed workflow result requires output_envelope"
5121                            .to_string(),
5122                    )
5123                });
5124            }
5125            if description.is_terminal() {
5126                let outcome =
5127                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5128                return Err(match outcome.kind {
5129                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5130                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5131                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5132                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5133                });
5134            }
5135            if started.elapsed() >= options.timeout {
5136                return Err(Error::Timeout);
5137            }
5138            tokio::time::sleep(options.poll_interval).await;
5139        }
5140    }
5141
5142    async fn result_target(
5143        &self,
5144        options: WorkflowResultOptions,
5145        selected_run_id: Option<&str>,
5146    ) -> Result<Value> {
5147        let started = Instant::now();
5148
5149        loop {
5150            let description = match selected_run_id {
5151                Some(run_id) => {
5152                    self.client
5153                        .describe_workflow_run(&self.workflow_id, run_id)
5154                        .await?
5155                }
5156                None => self.describe().await?,
5157            };
5158            if description.is_completed() {
5159                return Ok(description.output.unwrap_or(Value::Null));
5160            }
5161
5162            if description.is_terminal() {
5163                let outcome =
5164                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5165                return Err(match outcome.kind {
5166                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5167                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5168                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5169                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5170                });
5171            }
5172
5173            if started.elapsed() >= options.timeout {
5174                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
5175                    kind: WorkflowTerminalKind::TimedOut,
5176                    workflow_id: description
5177                        .workflow_id
5178                        .clone()
5179                        .unwrap_or_else(|| self.workflow_id.clone()),
5180                    run_id: description
5181                        .run_id
5182                        .clone()
5183                        .or_else(|| selected_run_id.map(str::to_string)),
5184                    reason: "result_wait_timeout".to_string(),
5185                    failure_category: Some("client_timeout".to_string()),
5186                    failure_id: None,
5187                    exception_type: None,
5188                    exception_class: None,
5189                    non_retryable: None,
5190                    message: Some(format!(
5191                        "workflow result was not terminal within {:?}",
5192                        options.timeout
5193                    )),
5194                    exception: None,
5195                    raw: description.raw_value(),
5196                }));
5197            }
5198
5199            tokio::time::sleep(options.poll_interval).await;
5200        }
5201    }
5202}
5203
5204#[derive(Clone, Copy, Debug)]
5205pub struct WorkflowResultOptions {
5206    pub poll_interval: Duration,
5207    pub timeout: Duration,
5208}
5209
5210impl Default for WorkflowResultOptions {
5211    fn default() -> Self {
5212        Self {
5213            poll_interval: Duration::from_millis(500),
5214            timeout: Duration::from_secs(30),
5215        }
5216    }
5217}
5218
5219#[derive(Clone, Debug, Deserialize)]
5220pub struct WorkflowDescription {
5221    pub workflow_id: Option<String>,
5222    pub run_id: Option<String>,
5223    pub workflow_type: Option<String>,
5224    pub status: Option<String>,
5225    #[serde(default)]
5226    pub closed_reason: Option<String>,
5227    #[serde(default)]
5228    pub error: Option<String>,
5229    #[serde(default)]
5230    pub failure: Option<Value>,
5231    #[serde(default)]
5232    pub exception: Option<Value>,
5233    #[serde(default)]
5234    pub failures: Vec<Value>,
5235    #[serde(default)]
5236    pub output: Option<Value>,
5237    #[serde(default)]
5238    pub output_envelope: Option<Value>,
5239    #[serde(skip)]
5240    pub output_avro_value: Option<AvroValue>,
5241    #[serde(flatten)]
5242    pub raw: HashMap<String, Value>,
5243}
5244
5245/// Lifecycle and backlog metadata for one run-scoped Workflow Stream.
5246#[derive(Clone, Debug, Deserialize)]
5247pub struct WorkflowStreamDescription {
5248    pub stream_name: String,
5249    pub status: String,
5250    pub last_offset: i64,
5251    pub total_items: u64,
5252    pub pending_items: u64,
5253    #[serde(default)]
5254    pub opened_at: Option<String>,
5255    #[serde(default)]
5256    pub last_appended_at: Option<String>,
5257    #[serde(default)]
5258    pub closed_at: Option<String>,
5259    #[serde(default)]
5260    pub error_reason: Option<String>,
5261    #[serde(default)]
5262    pub retention_seconds: Option<u64>,
5263    #[serde(flatten)]
5264    pub raw: HashMap<String, Value>,
5265}
5266
5267impl WorkflowStreamDescription {
5268    pub fn is_terminal(&self) -> bool {
5269        matches!(self.status.as_str(), "closed" | "errored")
5270    }
5271}
5272
5273/// One item for direct or replay-safe append.
5274#[derive(Clone, Debug, Default)]
5275pub struct WorkflowStreamAppendItem {
5276    pub payload_envelope: Option<Value>,
5277    pub payload_reference: Option<String>,
5278    pub item_type: Option<String>,
5279    pub content_type: Option<String>,
5280    pub idempotency_key: Option<String>,
5281}
5282
5283impl WorkflowStreamAppendItem {
5284    /// Encode an inline payload with the SDK's fixed Avro Value envelope.
5285    pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5286        let value = AvroValue::from_serialize(&payload)?;
5287        Ok(Self {
5288            payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5289            ..Self::default()
5290        })
5291    }
5292
5293    /// Preserve an external payload URI as an opaque service-contract reference.
5294    pub fn from_reference(reference: impl Into<String>) -> Self {
5295        Self {
5296            payload_reference: Some(reference.into()),
5297            ..Self::default()
5298        }
5299    }
5300
5301    pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5302        self.item_type = Some(item_type.into());
5303        self
5304    }
5305
5306    pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5307        self.content_type = Some(content_type.into());
5308        self
5309    }
5310
5311    pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5312        self.idempotency_key = Some(idempotency_key.into());
5313        self
5314    }
5315
5316    fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5317        let mut item = serde_json::Map::new();
5318        if let Some(payload) = &self.payload_envelope {
5319            item.insert("payload".to_string(), payload.clone());
5320            item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5321        }
5322        if let Some(reference) = &self.payload_reference {
5323            item.insert("payload_reference".to_string(), json!(reference));
5324        }
5325        if let Some(item_type) = &self.item_type {
5326            item.insert("item_type".to_string(), json!(item_type));
5327        }
5328        if let Some(content_type) = &self.content_type {
5329            item.insert("content_type".to_string(), json!(content_type));
5330        }
5331        if let Some(key) = derived_idempotency_key
5332            .as_ref()
5333            .or(self.idempotency_key.as_ref())
5334        {
5335            item.insert("idempotency_key".to_string(), json!(key));
5336        }
5337        Value::Object(item)
5338    }
5339}
5340
5341/// One durable item at its stable zero-based offset.
5342#[derive(Clone, Debug)]
5343pub struct WorkflowStreamItem {
5344    pub offset: u64,
5345    pub payload: Option<Value>,
5346    pub payload_envelope: Option<Value>,
5347    pub payload_reference: Option<String>,
5348    pub payload_codec: Option<String>,
5349    pub idempotency_key: Option<String>,
5350    pub item_type: Option<String>,
5351    pub content_type: Option<String>,
5352    pub origin: Option<String>,
5353    pub origin_reference: Option<String>,
5354    pub emitted_at: Option<String>,
5355    pub raw: Value,
5356}
5357
5358/// One bounded at-least-once subscription page.
5359#[derive(Clone, Debug)]
5360pub struct WorkflowStreamPage {
5361    pub stream: WorkflowStreamDescription,
5362    pub items: Vec<WorkflowStreamItem>,
5363    pub next_offset: u64,
5364    pub terminal: bool,
5365}
5366
5367/// Durable acceptance and deduplication outcome for an append request.
5368#[derive(Clone, Debug)]
5369pub struct WorkflowStreamAppendResult {
5370    pub stream: WorkflowStreamDescription,
5371    pub accepted_offsets: Vec<u64>,
5372    pub accepted: u64,
5373    pub deduped: u64,
5374}
5375
5376#[derive(Deserialize)]
5377struct WorkflowStreamListResponse {
5378    #[serde(default)]
5379    streams: Vec<WorkflowStreamDescription>,
5380}
5381
5382#[derive(Deserialize)]
5383struct WorkflowStreamDescriptionResponse {
5384    stream: WorkflowStreamDescription,
5385}
5386
5387#[derive(Deserialize)]
5388struct WorkflowStreamPageResponse {
5389    stream: WorkflowStreamDescription,
5390    #[serde(default)]
5391    items: Vec<Value>,
5392    next_offset: u64,
5393    terminal: bool,
5394}
5395
5396#[derive(Deserialize)]
5397struct WorkflowStreamAppendResponse {
5398    stream: WorkflowStreamDescription,
5399    #[serde(default)]
5400    accepted_offsets: Vec<u64>,
5401    accepted: u64,
5402    deduped: u64,
5403}
5404
5405impl WorkflowDescription {
5406    pub fn is_completed(&self) -> bool {
5407        matches!(self.status.as_deref(), Some("completed" | "Completed"))
5408    }
5409
5410    pub fn is_terminal(&self) -> bool {
5411        matches!(
5412            self.status.as_deref(),
5413            Some(
5414                "completed"
5415                    | "Completed"
5416                    | "failed"
5417                    | "Failed"
5418                    | "cancelled"
5419                    | "Cancelled"
5420                    | "terminated"
5421                    | "Terminated"
5422                    | "timed_out"
5423                    | "TimedOut",
5424            )
5425        )
5426    }
5427
5428    fn decode_payloads(&mut self) -> Result<()> {
5429        if let Some(envelope) = &self.output_envelope {
5430            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5431            self.output = Some(value.clone().into_json()?);
5432            self.output_avro_value = Some(value);
5433        }
5434
5435        Ok(())
5436    }
5437
5438    fn raw_value(&self) -> Value {
5439        let mut data = self.raw.clone();
5440        data.insert(
5441            "workflow_id".to_string(),
5442            self.workflow_id
5443                .clone()
5444                .map(Value::String)
5445                .unwrap_or(Value::Null),
5446        );
5447        data.insert(
5448            "run_id".to_string(),
5449            self.run_id
5450                .clone()
5451                .map(Value::String)
5452                .unwrap_or(Value::Null),
5453        );
5454        data.insert(
5455            "workflow_type".to_string(),
5456            self.workflow_type
5457                .clone()
5458                .map(Value::String)
5459                .unwrap_or(Value::Null),
5460        );
5461        data.insert(
5462            "status".to_string(),
5463            self.status
5464                .clone()
5465                .map(Value::String)
5466                .unwrap_or(Value::Null),
5467        );
5468        data.insert(
5469            "closed_reason".to_string(),
5470            self.closed_reason
5471                .clone()
5472                .map(Value::String)
5473                .unwrap_or(Value::Null),
5474        );
5475        if let Some(failure) = &self.failure {
5476            data.insert("failure".to_string(), failure.clone());
5477        }
5478        if let Some(exception) = &self.exception {
5479            data.insert("exception".to_string(), exception.clone());
5480        }
5481        Value::Object(data.into_iter().collect())
5482    }
5483}
5484
5485fn workflow_terminal_outcome(
5486    description: &WorkflowDescription,
5487    workflow_id: &str,
5488    run_id: Option<&str>,
5489) -> WorkflowTerminalOutcome {
5490    let terminal_kind = description
5491        .closed_reason
5492        .as_deref()
5493        .or(description.status.as_deref())
5494        .unwrap_or("failed")
5495        .to_ascii_lowercase();
5496    let kind = match terminal_kind.as_str() {
5497        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5498        "terminated" => WorkflowTerminalKind::Terminated,
5499        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5500        _ => WorkflowTerminalKind::Failed,
5501    };
5502    let default_reason = match kind {
5503        WorkflowTerminalKind::Failed => "workflow_failed",
5504        WorkflowTerminalKind::Cancelled => "cancelled",
5505        WorkflowTerminalKind::Terminated => "terminated",
5506        WorkflowTerminalKind::TimedOut => "timed_out",
5507    };
5508    let failure = description
5509        .failure
5510        .as_ref()
5511        .filter(|value| value.is_object());
5512    let nested_failure = failure
5513        .and_then(|value| value.get("failures"))
5514        .and_then(Value::as_array)
5515        .and_then(|failures| failures.last())
5516        .or_else(|| description.failures.last());
5517    let exception = description
5518        .exception
5519        .clone()
5520        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5521        .or_else(|| {
5522            nested_failure
5523                .and_then(|value| value.get("exception_payload"))
5524                .cloned()
5525        });
5526    let string_field = |name: &str| {
5527        failure
5528            .and_then(|value| value.get(name))
5529            .and_then(Value::as_str)
5530            .or_else(|| {
5531                nested_failure
5532                    .and_then(|value| value.get(name))
5533                    .and_then(Value::as_str)
5534            })
5535            .map(str::to_string)
5536    };
5537    let exception_field = |name: &str| {
5538        exception
5539            .as_ref()
5540            .and_then(|value| value.get(name))
5541            .and_then(Value::as_str)
5542            .map(str::to_string)
5543    };
5544    let message = description
5545        .error
5546        .clone()
5547        .or_else(|| string_field("message"))
5548        .or_else(|| exception_field("message"));
5549    let reason = description
5550        .raw
5551        .get("reason")
5552        .and_then(Value::as_str)
5553        .map(str::to_string)
5554        .or_else(|| {
5555            failure
5556                .and_then(|value| value.get("reason"))
5557                .and_then(Value::as_str)
5558                .map(str::to_string)
5559        })
5560        .or_else(|| description.closed_reason.clone())
5561        .unwrap_or_else(|| default_reason.to_string());
5562    let failure_id = string_field("failure_id").or_else(|| {
5563        nested_failure
5564            .and_then(|value| value.get("id"))
5565            .and_then(Value::as_str)
5566            .map(str::to_string)
5567    });
5568
5569    WorkflowTerminalOutcome {
5570        kind,
5571        workflow_id: description
5572            .workflow_id
5573            .clone()
5574            .unwrap_or_else(|| workflow_id.to_string()),
5575        run_id: description
5576            .run_id
5577            .clone()
5578            .or_else(|| run_id.map(str::to_string)),
5579        reason,
5580        failure_category: string_field("failure_category")
5581            .or_else(|| Some(default_reason.to_string())),
5582        failure_id,
5583        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5584        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5585        non_retryable: failure
5586            .and_then(|value| value.get("non_retryable"))
5587            .and_then(Value::as_bool)
5588            .or_else(|| {
5589                nested_failure
5590                    .and_then(|value| value.get("non_retryable"))
5591                    .and_then(Value::as_bool)
5592            }),
5593        message,
5594        exception,
5595        raw: description.raw_value(),
5596    }
5597}
5598
5599#[derive(Clone, Debug, Deserialize)]
5600pub struct RegisterWorkerResponse {
5601    pub worker_id: String,
5602    pub registered: bool,
5603    #[serde(default)]
5604    pub heartbeat_interval_seconds: Option<u64>,
5605    #[serde(default)]
5606    pub protocol_version: Option<String>,
5607    #[serde(default)]
5608    pub server_capabilities: Option<Value>,
5609}
5610
5611/// Result of gracefully removing a worker-plane registration.
5612#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5613pub struct WorkerDeregistrationEnvelope {
5614    pub worker_id: String,
5615    pub outcome: String,
5616    pub recovered_workflow_task_count: u64,
5617}
5618
5619#[derive(Clone, Debug, Deserialize)]
5620pub struct PollWorkflowTaskResponse {
5621    #[serde(default)]
5622    pub task: Option<WorkflowTask>,
5623    #[serde(default)]
5624    pub poll_status: Option<String>,
5625    #[serde(default)]
5626    pub reason: Option<String>,
5627    #[serde(default)]
5628    pub protocol_version: Option<String>,
5629    #[serde(default)]
5630    pub server_capabilities: Option<Value>,
5631}
5632
5633impl PollWorkflowTaskResponse {
5634    /// Classify this response without parsing server display text.
5635    pub fn outcome(&self) -> WorkerPollOutcome {
5636        worker_poll_outcome(
5637            self.task.is_some(),
5638            self.poll_status.as_deref(),
5639            self.reason.as_deref(),
5640        )
5641    }
5642}
5643
5644fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5645    let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5646        return false;
5647    };
5648    let supported = capabilities
5649        .get("workflow_memo_updates")
5650        .and_then(Value::as_object)
5651        .and_then(|memo| memo.get("supported"))
5652        .and_then(Value::as_bool)
5653        == Some(true);
5654    let command_advertised = capabilities
5655        .get("supported_workflow_task_commands")
5656        .and_then(Value::as_array)
5657        .is_some_and(|commands| {
5658            commands
5659                .iter()
5660                .any(|command| command.as_str() == Some("upsert_memo"))
5661        });
5662    supported && command_advertised
5663}
5664
5665fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5666    commands
5667        .iter()
5668        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5669}
5670
5671#[derive(Clone, Debug, Deserialize)]
5672pub struct PollActivityTaskResponse {
5673    #[serde(default)]
5674    pub task: Option<ActivityTask>,
5675    #[serde(default)]
5676    pub poll_status: Option<String>,
5677    #[serde(default)]
5678    pub reason: Option<String>,
5679}
5680
5681impl PollActivityTaskResponse {
5682    /// Classify this response without parsing server display text.
5683    pub fn outcome(&self) -> WorkerPollOutcome {
5684        worker_poll_outcome(
5685            self.task.is_some(),
5686            self.poll_status.as_deref(),
5687            self.reason.as_deref(),
5688        )
5689    }
5690}
5691
5692#[derive(Clone, Debug, Deserialize)]
5693pub struct PollQueryTaskResponse {
5694    #[serde(default)]
5695    pub task: Option<QueryTask>,
5696    #[serde(default)]
5697    pub poll_status: Option<String>,
5698    #[serde(default)]
5699    pub reason: Option<String>,
5700}
5701
5702impl PollQueryTaskResponse {
5703    /// Classify this response without parsing server display text.
5704    pub fn outcome(&self) -> WorkerPollOutcome {
5705        worker_poll_outcome(
5706            self.task.is_some(),
5707            self.poll_status.as_deref(),
5708            self.reason.as_deref(),
5709        )
5710    }
5711}
5712
5713/// Stable classification for worker poll responses.
5714#[derive(Clone, Debug, PartialEq, Eq)]
5715pub enum WorkerPollOutcome {
5716    /// A task was leased and is available on the response.
5717    Task,
5718    /// No task was leased, but the worker should continue polling.
5719    Idle {
5720        poll_status: Option<String>,
5721        reason: Option<String>,
5722    },
5723    /// The server asked this worker to stop claiming new work.
5724    Stop {
5725        poll_status: Option<String>,
5726        reason: Option<String>,
5727    },
5728}
5729
5730impl WorkerPollOutcome {
5731    pub fn should_stop(&self) -> bool {
5732        matches!(self, Self::Stop { .. })
5733    }
5734}
5735
5736fn worker_poll_outcome(
5737    has_task: bool,
5738    poll_status: Option<&str>,
5739    reason: Option<&str>,
5740) -> WorkerPollOutcome {
5741    if worker_poll_is_stop(poll_status, reason) {
5742        return WorkerPollOutcome::Stop {
5743            poll_status: poll_status.map(str::to_string),
5744            reason: reason.map(str::to_string),
5745        };
5746    }
5747
5748    if has_task {
5749        WorkerPollOutcome::Task
5750    } else {
5751        WorkerPollOutcome::Idle {
5752            poll_status: poll_status.map(str::to_string),
5753            reason: reason.map(str::to_string),
5754        }
5755    }
5756}
5757
5758/// An ephemeral server-routed query task.
5759#[derive(Clone, Debug, Deserialize)]
5760pub struct QueryTask {
5761    pub query_task_id: String,
5762    #[serde(default = "default_workflow_task_attempt")]
5763    pub query_task_attempt: u64,
5764    #[serde(default)]
5765    pub lease_owner: Option<String>,
5766    #[serde(default)]
5767    pub workflow_id: Option<String>,
5768    #[serde(default)]
5769    pub run_id: Option<String>,
5770    pub workflow_type: String,
5771    pub query_name: String,
5772    #[serde(
5773        default = "missing_task_payload_codec",
5774        deserialize_with = "deserialize_task_payload_codec"
5775    )]
5776    pub payload_codec: String,
5777    #[serde(default)]
5778    pub workflow_arguments: Option<Value>,
5779    #[serde(default)]
5780    pub query_arguments: Option<Value>,
5781    #[serde(default)]
5782    pub history_events: Vec<HistoryEvent>,
5783    #[serde(default)]
5784    pub history_export: Option<Value>,
5785    #[serde(default)]
5786    pub run_status: Option<String>,
5787}
5788
5789#[derive(Clone, Debug, Deserialize)]
5790pub struct WorkflowTask {
5791    pub task_id: String,
5792    #[serde(default)]
5793    pub workflow_command_id: Option<String>,
5794    #[serde(default)]
5795    pub workflow_id: Option<String>,
5796    #[serde(default)]
5797    pub run_id: Option<String>,
5798    pub workflow_type: String,
5799    #[serde(default)]
5800    pub cancel_requested: bool,
5801    #[serde(
5802        default = "missing_task_payload_codec",
5803        deserialize_with = "deserialize_task_payload_codec"
5804    )]
5805    pub payload_codec: String,
5806    #[serde(default)]
5807    pub arguments: Option<Value>,
5808    #[serde(default)]
5809    pub history_events: Vec<HistoryEvent>,
5810    #[serde(default)]
5811    pub total_history_events: Option<u64>,
5812    #[serde(default)]
5813    pub history_size_bytes: Option<u64>,
5814    #[serde(default)]
5815    pub continue_as_new_recommended: Option<bool>,
5816    #[serde(default)]
5817    pub history_budget_pressure: Option<String>,
5818    #[serde(default)]
5819    pub next_history_page_token: Option<String>,
5820    #[serde(default = "default_workflow_task_attempt")]
5821    pub workflow_task_attempt: u64,
5822    #[serde(default)]
5823    pub workflow_signal_id: Option<String>,
5824    #[serde(default)]
5825    pub signal_name: Option<String>,
5826    #[serde(default)]
5827    pub signal_arguments: Option<Value>,
5828    #[serde(default)]
5829    pub workflow_update_id: Option<String>,
5830    #[serde(default)]
5831    pub update_name: Option<String>,
5832    #[serde(default)]
5833    pub lease_owner: Option<String>,
5834}
5835
5836impl WorkflowTask {
5837    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5838        self.history_events.extend(page.history_events);
5839
5840        if page.total_history_events.is_some() {
5841            self.total_history_events = page.total_history_events;
5842        }
5843
5844        self.next_history_page_token = page
5845            .next_history_page_token
5846            .filter(|token| !token.is_empty());
5847    }
5848}
5849
5850#[derive(Clone, Debug, Deserialize)]
5851struct WorkflowTaskHistoryPage {
5852    #[serde(default)]
5853    history_events: Vec<HistoryEvent>,
5854    #[serde(default)]
5855    total_history_events: Option<u64>,
5856    #[serde(default)]
5857    next_history_page_token: Option<String>,
5858}
5859
5860#[derive(Clone, Debug, Deserialize)]
5861pub struct ActivityTask {
5862    pub task_id: String,
5863    #[serde(default)]
5864    pub activity_attempt_id: Option<String>,
5865    #[serde(default)]
5866    pub attempt_id: Option<String>,
5867    pub activity_type: String,
5868    #[serde(
5869        default = "missing_task_payload_codec",
5870        deserialize_with = "deserialize_task_payload_codec"
5871    )]
5872    pub payload_codec: String,
5873    #[serde(default)]
5874    pub arguments: Option<Value>,
5875    #[serde(default = "default_attempt_number")]
5876    pub attempt_number: u64,
5877    #[serde(default)]
5878    pub lease_owner: Option<String>,
5879}
5880
5881#[derive(Clone, Debug, Deserialize)]
5882pub struct HistoryEvent {
5883    #[serde(alias = "type")]
5884    pub event_type: String,
5885    #[serde(default)]
5886    pub payload: Value,
5887    #[serde(flatten)]
5888    pub raw: HashMap<String, Value>,
5889}
5890
5891/// One decoded signal in the committed workflow-history snapshot.
5892#[derive(Clone, Debug, PartialEq)]
5893pub struct QuerySignal {
5894    pub id: Option<String>,
5895    pub name: String,
5896    pub arguments: Vec<Value>,
5897    avro_arguments: Vec<AvroValue>,
5898    pub workflow_sequence: Option<u64>,
5899}
5900
5901impl QuerySignal {
5902    /// Lossless fixed Avro Value arguments for this committed signal.
5903    pub fn arguments_avro_value(&self) -> &[AvroValue] {
5904        &self.avro_arguments
5905    }
5906}
5907
5908/// Immutable state supplied to a registered query handler.
5909///
5910/// This context intentionally exposes no activity, signal-wait, or command
5911/// APIs. Query handlers inspect committed history and return a value; query
5912/// completion does not append an event or advance deterministic execution.
5913#[derive(Clone, Debug)]
5914pub struct QueryContext {
5915    pub workflow_id: Option<String>,
5916    pub run_id: Option<String>,
5917    pub workflow_type: String,
5918    pub run_status: Option<String>,
5919    workflow_input: Value,
5920    workflow_input_avro_value: AvroValue,
5921    history_events: Arc<Vec<HistoryEvent>>,
5922    signal_events: Arc<Vec<QuerySignal>>,
5923}
5924
5925impl QueryContext {
5926    /// The normalized argument list used to start the workflow.
5927    pub fn workflow_input(&self) -> &Value {
5928        &self.workflow_input
5929    }
5930
5931    /// The lossless fixed Avro Value argument list used to start the workflow.
5932    pub fn workflow_input_avro_value(&self) -> &AvroValue {
5933        &self.workflow_input_avro_value
5934    }
5935
5936    /// The immutable committed history used for this query snapshot.
5937    pub fn history_events(&self) -> &[HistoryEvent] {
5938        self.history_events.as_slice()
5939    }
5940
5941    /// All decoded signals in committed workflow order.
5942    pub fn signal_events(&self) -> &[QuerySignal] {
5943        self.signal_events.as_slice()
5944    }
5945
5946    /// Decoded argument lists for each committed signal with `signal_name`.
5947    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5948        self.signal_events
5949            .iter()
5950            .filter(|signal| signal.name == signal_name)
5951            .map(|signal| signal.arguments.clone())
5952            .collect()
5953    }
5954
5955    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
5956    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5957        self.signal_events
5958            .iter()
5959            .filter(|signal| signal.name == signal_name)
5960            .map(|signal| signal.avro_arguments.clone())
5961            .collect()
5962    }
5963}
5964
5965#[derive(Clone, Debug, Deserialize)]
5966pub struct ActivityHeartbeatResponse {
5967    #[serde(default)]
5968    pub cancel_requested: bool,
5969    #[serde(default)]
5970    pub heartbeat_recorded: bool,
5971    #[serde(default)]
5972    pub can_continue: Option<bool>,
5973    #[serde(default)]
5974    pub reason: Option<String>,
5975    #[serde(default)]
5976    pub run_closed_reason: Option<String>,
5977    #[serde(default)]
5978    pub run_closed_at: Option<String>,
5979    #[serde(default)]
5980    pub lease_expires_at: Option<String>,
5981    #[serde(default)]
5982    pub last_heartbeat_at: Option<String>,
5983}
5984
5985impl ActivityHeartbeatResponse {
5986    /// Whether the activity should stop instead of attempting completion.
5987    pub fn should_stop(&self) -> bool {
5988        self.cancel_requested || self.can_continue == Some(false)
5989    }
5990}
5991
5992fn missing_task_payload_codec() -> String {
5993    MISSING_TASK_PAYLOAD_CODEC.to_string()
5994}
5995
5996fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5997where
5998    D: Deserializer<'de>,
5999{
6000    Ok(match Value::deserialize(deserializer)? {
6001        Value::String(codec) => codec,
6002        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
6003        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
6004    })
6005}
6006
6007fn default_workflow_task_attempt() -> u64 {
6008    1
6009}
6010
6011fn default_attempt_number() -> u64 {
6012    1
6013}
6014
6015type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6016type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
6017type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
6018type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
6019type ReplayedWorkflowHandler =
6020    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
6021type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6022type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
6023type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6024type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
6025type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
6026type ReplayedQueryHandler = Arc<
6027    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
6028        + Send
6029        + Sync,
6030>;
6031type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
6032
6033struct ReplayedWorkflowInvocation {
6034    future: WorkflowFuture,
6035    snapshot: WorkflowStateSnapshot,
6036}
6037
6038#[derive(Clone)]
6039struct RegisteredWorkflow {
6040    execute: WorkflowHandler,
6041    replay: Option<ReplayedWorkflowHandler>,
6042    state_type: Option<TypeId>,
6043    definition_fingerprint: Option<String>,
6044}
6045
6046#[derive(Debug)]
6047struct WorkflowTaskDecision {
6048    commands: Vec<Value>,
6049    message_stream_cursors: Vec<Value>,
6050    message_stream_waits: Vec<Value>,
6051}
6052
6053impl WorkflowTaskDecision {
6054    fn without_message_streams(commands: Vec<Value>) -> Self {
6055        Self {
6056            commands,
6057            message_stream_cursors: Vec::new(),
6058            message_stream_waits: Vec::new(),
6059        }
6060    }
6061}
6062
6063#[derive(Clone)]
6064enum RegisteredQuery {
6065    Snapshot(QueryHandler),
6066    Replayed {
6067        state_type: TypeId,
6068        handler: ReplayedQueryHandler,
6069    },
6070}
6071
6072#[derive(Clone, Debug)]
6073pub struct WorkerHeartbeatObservation {
6074    pub worker_id: String,
6075    pub task_queue: String,
6076    pub acknowledged_at_unix_millis: u64,
6077    pub acknowledgement: Value,
6078}
6079
6080/// Bounded retry policy for worker poll acquisition and worker heartbeats.
6081///
6082/// Expected empty long polls and explicit Server capacity backpressure are
6083/// normal worker states. Capacity backpressure honors the advertised retry
6084/// delay without consuming this retry budget. Other transport failures,
6085/// retryable HTTP 408/429 responses, and server errors are retried with capped
6086/// exponential backoff. Authentication, protocol, codec, and handler failures
6087/// are never retried by the worker.
6088/// Explicit storage admission pauses also preserve the already-serialized worker
6089/// request without consuming this budget or rerunning its handler. The delay is
6090/// capped by `max_backoff`, and shutdown interrupts storage waits.
6091#[derive(Clone, Copy, Debug)]
6092pub struct WorkerRetryPolicy {
6093    /// Number of retries after the initial request fails.
6094    pub max_retries: usize,
6095    /// Delay before the first retry.
6096    pub initial_backoff: Duration,
6097    /// Maximum delay between retries.
6098    pub max_backoff: Duration,
6099}
6100
6101impl Default for WorkerRetryPolicy {
6102    fn default() -> Self {
6103        Self {
6104            max_retries: 5,
6105            initial_backoff: Duration::from_millis(100),
6106            max_backoff: Duration::from_secs(5),
6107        }
6108    }
6109}
6110
6111#[derive(Clone, Debug)]
6112struct WorkerStorageAdmission {
6113    policy: WorkerRetryPolicy,
6114    stop: Arc<AtomicBool>,
6115}
6116
6117struct StopWorkerOnDrop(Arc<AtomicBool>);
6118
6119impl Drop for StopWorkerOnDrop {
6120    fn drop(&mut self) {
6121        self.0.store(true, Ordering::SeqCst);
6122    }
6123}
6124
6125async fn wait_for_worker_stop(stop: &AtomicBool) {
6126    while !stop.load(Ordering::SeqCst) {
6127        tokio::time::sleep(Duration::from_millis(100)).await;
6128    }
6129}
6130
6131#[derive(Clone, Copy, Debug, PartialEq, Eq)]
6132enum ManagedPollOutcome {
6133    Idle,
6134    Handled,
6135    Stop,
6136}
6137
6138#[derive(Clone)]
6139pub struct Worker {
6140    client: Client,
6141    worker_id: String,
6142    task_queue: String,
6143    workflows: HashMap<String, RegisteredWorkflow>,
6144    activities: HashMap<String, ActivityHandler>,
6145    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
6146    updates: HashMap<String, HashMap<String, UpdateHandler>>,
6147    max_concurrent_workflow_tasks: usize,
6148    max_concurrent_activity_tasks: usize,
6149    poll_timeout: Duration,
6150    heartbeat_interval: Duration,
6151    retry_policy: WorkerRetryPolicy,
6152    heartbeat_observer: Option<WorkerHeartbeatObserver>,
6153}
6154
6155impl Worker {
6156    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
6157        Self {
6158            client,
6159            worker_id: default_worker_id(),
6160            task_queue: task_queue.into(),
6161            workflows: HashMap::new(),
6162            activities: HashMap::new(),
6163            queries: HashMap::new(),
6164            updates: HashMap::new(),
6165            max_concurrent_workflow_tasks: 10,
6166            max_concurrent_activity_tasks: 10,
6167            poll_timeout: Duration::from_secs(30),
6168            heartbeat_interval: Duration::from_secs(60),
6169            retry_policy: WorkerRetryPolicy::default(),
6170            heartbeat_observer: None,
6171        }
6172    }
6173
6174    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
6175        self.worker_id = worker_id.into();
6176        self
6177    }
6178
6179    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
6180        self.poll_timeout = timeout;
6181        self
6182    }
6183
6184    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
6185        self.heartbeat_interval = interval;
6186        self
6187    }
6188
6189    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
6190    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
6191        self.retry_policy = policy;
6192        self
6193    }
6194
6195    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
6196    where
6197        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
6198    {
6199        self.heartbeat_observer = Some(Arc::new(observer));
6200        self
6201    }
6202
6203    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
6204        self.max_concurrent_workflow_tasks = count.max(1);
6205        self
6206    }
6207
6208    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
6209        self.max_concurrent_activity_tasks = count.max(1);
6210        self
6211    }
6212
6213    /// Register a workflow handler.
6214    ///
6215    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
6216    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
6217    /// while acquiring or decoding a worker task remain worker-operation
6218    /// failures and do not get converted into workflow outcomes.
6219    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
6220    where
6221        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
6222        Fut: Future<Output = Result<Value>> + Send + 'static,
6223    {
6224        let handler = Arc::new(handler);
6225        self.workflows.insert(
6226            workflow_type.into(),
6227            RegisteredWorkflow {
6228                execute: Arc::new(move |ctx, input| {
6229                    let handler = Arc::clone(&handler);
6230                    Box::pin(async move {
6231                        let result = handler(ctx, input.into_json()?).await?;
6232                        AvroValue::from_serialize(&result)
6233                    })
6234                }),
6235                replay: None,
6236                state_type: None,
6237                definition_fingerprint: None,
6238            },
6239        );
6240    }
6241
6242    /// Register a workflow with one Serde request value and a Serde result.
6243    ///
6244    /// This is an ergonomic adapter over the same fixed Avro Value protocol as
6245    /// [`Worker::register_workflow_avro_value`]. It does not create or publish a
6246    /// workflow-specific schema. A task must contain zero arguments for a unit
6247    /// request or exactly one argument for every other request type.
6248    ///
6249    /// See the runnable
6250    /// [`hello_world` example](https://github.com/durable-workflow/sdk-rust/blob/main/examples/hello_world.rs)
6251    /// for typed workflow and activity contracts with retry and timeout policy.
6252    pub fn register_typed_workflow<I, O, F, Fut>(
6253        &mut self,
6254        workflow_type: impl Into<String>,
6255        handler: F,
6256    ) where
6257        I: DeserializeOwned + Send + 'static,
6258        O: Serialize + Send + 'static,
6259        F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6260        Fut: Future<Output = Result<O>> + Send + 'static,
6261    {
6262        let workflow_type = workflow_type.into();
6263        let handler_name = workflow_type.clone();
6264        let handler = Arc::new(handler);
6265        self.workflows.insert(
6266            workflow_type,
6267            RegisteredWorkflow {
6268                execute: Arc::new(move |ctx, input| {
6269                    let handler = Arc::clone(&handler);
6270                    let handler_name = handler_name.clone();
6271                    Box::pin(async move {
6272                        let input =
6273                            decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6274                        let result = handler(ctx, input).await?;
6275                        encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6276                    })
6277                }),
6278                replay: None,
6279                state_type: None,
6280                definition_fingerprint: None,
6281            },
6282        );
6283    }
6284
6285    /// Register a workflow on the lossless fixed Avro Value surface.
6286    pub fn register_workflow_avro_value<F, Fut>(
6287        &mut self,
6288        workflow_type: impl Into<String>,
6289        handler: F,
6290    ) where
6291        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6292        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6293    {
6294        self.workflows.insert(
6295            workflow_type.into(),
6296            RegisteredWorkflow {
6297                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6298                replay: None,
6299                state_type: None,
6300                definition_fingerprint: None,
6301            },
6302        );
6303    }
6304
6305    /// Register a workflow whose typed instance state can be reconstructed for queries.
6306    ///
6307    /// `state_factory` creates a fresh instance for every normal workflow task and
6308    /// query replay. The workflow handler is the single source of truth for state
6309    /// transitions: it updates [`WorkflowInstance`] after activities and signals
6310    /// resolve. Query replay runs this same handler over committed history and
6311    /// discards any commands it would emit.
6312    pub fn register_replayed_workflow<S, Factory, F, Fut>(
6313        &mut self,
6314        workflow_type: impl Into<String>,
6315        state_factory: Factory,
6316        handler: F,
6317    ) where
6318        S: Clone + Send + Sync + 'static,
6319        Factory: Fn() -> S + Send + Sync + 'static,
6320        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6321        Fut: Future<Output = Result<Value>> + Send + 'static,
6322    {
6323        let state_factory = Arc::new(state_factory);
6324        let handler = Arc::new(handler);
6325
6326        let execute_factory = Arc::clone(&state_factory);
6327        let execute_handler = Arc::clone(&handler);
6328        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6329            let state = WorkflowInstance::new(execute_factory());
6330            let handler = Arc::clone(&execute_handler);
6331            Box::pin(async move {
6332                let result = handler(ctx, input.into_json()?, state).await?;
6333                AvroValue::from_serialize(&result)
6334            }) as WorkflowFuture
6335        });
6336
6337        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6338            let state = WorkflowInstance::new(state_factory());
6339            let snapshot_state = state.clone();
6340            let snapshot: WorkflowStateSnapshot =
6341                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6342            let replay_handler = Arc::clone(&handler);
6343            let future = async move {
6344                let result = replay_handler(ctx, input.into_json()?, state).await?;
6345                AvroValue::from_serialize(&result)
6346            };
6347            ReplayedWorkflowInvocation {
6348                future: Box::pin(future),
6349                snapshot,
6350            }
6351        });
6352
6353        self.workflows.insert(
6354            workflow_type.into(),
6355            RegisteredWorkflow {
6356                execute,
6357                replay: Some(replay),
6358                state_type: Some(TypeId::of::<S>()),
6359                definition_fingerprint: None,
6360            },
6361        );
6362    }
6363
6364    /// Register a replayable workflow with one Serde request value and result.
6365    ///
6366    /// Normal task execution and instance-state query replay both decode and
6367    /// encode through the fixed Avro Value codec. The state factory and handler
6368    /// otherwise follow [`Worker::register_replayed_workflow`].
6369    pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6370        &mut self,
6371        workflow_type: impl Into<String>,
6372        state_factory: Factory,
6373        handler: F,
6374    ) where
6375        I: DeserializeOwned + Send + 'static,
6376        O: Serialize + Send + 'static,
6377        S: Clone + Send + Sync + 'static,
6378        Factory: Fn() -> S + Send + Sync + 'static,
6379        F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6380        Fut: Future<Output = Result<O>> + Send + 'static,
6381    {
6382        let workflow_type = workflow_type.into();
6383        let state_factory = Arc::new(state_factory);
6384        let handler = Arc::new(handler);
6385
6386        let execute_name = workflow_type.clone();
6387        let execute_factory = Arc::clone(&state_factory);
6388        let execute_handler = Arc::clone(&handler);
6389        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6390            let state = WorkflowInstance::new(execute_factory());
6391            let handler = Arc::clone(&execute_handler);
6392            let handler_name = execute_name.clone();
6393            Box::pin(async move {
6394                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6395                let result = handler(ctx, input, state).await?;
6396                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6397            }) as WorkflowFuture
6398        });
6399
6400        let replay_name = workflow_type.clone();
6401        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6402            let state = WorkflowInstance::new(state_factory());
6403            let snapshot_state = state.clone();
6404            let snapshot: WorkflowStateSnapshot =
6405                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6406            let handler = Arc::clone(&handler);
6407            let handler_name = replay_name.clone();
6408            let future = async move {
6409                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6410                let result = handler(ctx, input, state).await?;
6411                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6412            };
6413            ReplayedWorkflowInvocation {
6414                future: Box::pin(future),
6415                snapshot,
6416            }
6417        });
6418
6419        self.workflows.insert(
6420            workflow_type,
6421            RegisteredWorkflow {
6422                execute,
6423                replay: Some(replay),
6424                state_type: Some(TypeId::of::<S>()),
6425                definition_fingerprint: None,
6426            },
6427        );
6428    }
6429
6430    /// Bind a registered workflow to compile-time embedded source for safe redrive.
6431    ///
6432    /// Supply `include_str!` values for the workflow body and every helper whose
6433    /// behavior can affect replay. A handler re-registration clears the local
6434    /// identity; call this after registering the final handler. The server
6435    /// rejects reusing a worker ID when its prior fingerprint disappears.
6436    /// Workflows without source identity remain runnable but cannot be safely redriven.
6437    /// For example, pass `&[include_str!("workflows.rs")]` when the handler and
6438    /// replay-sensitive helpers are in a sibling `workflows.rs` file.
6439    pub fn set_workflow_definition_sources(
6440        &mut self,
6441        workflow_type: &str,
6442        sources: &[&str],
6443    ) -> Result<()> {
6444        if sources.is_empty() || sources.iter().any(|source| source.is_empty()) {
6445            return Err(Error::Codec(
6446                "workflow definition sources must be non-empty".to_string(),
6447            ));
6448        }
6449        let workflow = self.workflows.get_mut(workflow_type).ok_or_else(|| {
6450            Error::Codec(format!("workflow type {workflow_type:?} is not registered"))
6451        })?;
6452        let mut hasher = Sha256::new();
6453        hasher.update(b"durable-workflow-rust.workflow-definition.v1\0");
6454        hasher.update((workflow_type.len() as u64).to_be_bytes());
6455        hasher.update(workflow_type.as_bytes());
6456        for source in sources {
6457            hasher.update((source.len() as u64).to_be_bytes());
6458            hasher.update(source.as_bytes());
6459        }
6460        workflow.definition_fingerprint = Some(format!("sha256:{:x}", hasher.finalize()));
6461        Ok(())
6462    }
6463
6464    /// Register a replayable workflow on the lossless fixed Avro Value surface.
6465    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6466        &mut self,
6467        workflow_type: impl Into<String>,
6468        state_factory: Factory,
6469        handler: F,
6470    ) where
6471        S: Clone + Send + Sync + 'static,
6472        Factory: Fn() -> S + Send + Sync + 'static,
6473        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6474        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6475    {
6476        let state_factory = Arc::new(state_factory);
6477        let handler = Arc::new(handler);
6478
6479        let execute_factory = Arc::clone(&state_factory);
6480        let execute_handler = Arc::clone(&handler);
6481        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6482            let state = WorkflowInstance::new(execute_factory());
6483            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6484        });
6485
6486        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6487            let state = WorkflowInstance::new(state_factory());
6488            let snapshot_state = state.clone();
6489            let snapshot: WorkflowStateSnapshot =
6490                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6491            ReplayedWorkflowInvocation {
6492                future: Box::pin(handler(ctx, input, state)),
6493                snapshot,
6494            }
6495        });
6496
6497        self.workflows.insert(
6498            workflow_type.into(),
6499            RegisteredWorkflow {
6500                execute,
6501                replay: Some(replay),
6502                state_type: Some(TypeId::of::<S>()),
6503                definition_fingerprint: None,
6504            },
6505        );
6506    }
6507
6508    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6509    where
6510        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6511        Fut: Future<Output = Result<Value>> + Send + 'static,
6512    {
6513        let handler = Arc::new(handler);
6514        self.activities.insert(
6515            activity_type.into(),
6516            Arc::new(move |ctx, args| {
6517                let handler = Arc::clone(&handler);
6518                Box::pin(async move {
6519                    let result = handler(ctx, args.into_json()?).await?;
6520                    AvroValue::from_serialize(&result)
6521                })
6522            }),
6523        );
6524    }
6525
6526    /// Register an activity with one Serde request value and a Serde result.
6527    ///
6528    /// Inputs and results use the platform's fixed Avro Value schema. Shape
6529    /// mismatches and unsupported Serde values return [`Error::HandlerType`]
6530    /// with the activity name and Rust type.
6531    pub fn register_typed_activity<I, O, F, Fut>(
6532        &mut self,
6533        activity_type: impl Into<String>,
6534        handler: F,
6535    ) where
6536        I: DeserializeOwned + Send + 'static,
6537        O: Serialize + Send + 'static,
6538        F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6539        Fut: Future<Output = Result<O>> + Send + 'static,
6540    {
6541        let activity_type = activity_type.into();
6542        let handler_name = activity_type.clone();
6543        let handler = Arc::new(handler);
6544        self.activities.insert(
6545            activity_type,
6546            Arc::new(move |ctx, input| {
6547                let handler = Arc::clone(&handler);
6548                let handler_name = handler_name.clone();
6549                Box::pin(async move {
6550                    let input =
6551                        decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6552                    let result = handler(ctx, input).await?;
6553                    encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6554                })
6555            }),
6556        );
6557    }
6558
6559    /// Register an activity on the lossless fixed Avro Value surface.
6560    pub fn register_activity_avro_value<F, Fut>(
6561        &mut self,
6562        activity_type: impl Into<String>,
6563        handler: F,
6564    ) where
6565        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6566        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6567    {
6568        self.activities.insert(
6569            activity_type.into(),
6570            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6571        );
6572    }
6573
6574    /// Register a named, read-only query handler for a workflow type.
6575    ///
6576    /// The workflow type must also be registered with [`Worker::register_workflow`]
6577    /// before the worker runs. The handler receives only an immutable committed
6578    /// state snapshot and normalized query arguments.
6579    pub fn register_query<F, Fut>(
6580        &mut self,
6581        workflow_type: impl Into<String>,
6582        query_name: impl Into<String>,
6583        handler: F,
6584    ) where
6585        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6586        Fut: Future<Output = Result<Value>> + Send + 'static,
6587    {
6588        let handler = Arc::new(handler);
6589        self.queries
6590            .entry(workflow_type.into())
6591            .or_default()
6592            .insert(
6593                query_name.into(),
6594                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6595                    let handler = Arc::clone(&handler);
6596                    Box::pin(async move {
6597                        let result = handler(ctx, args.into_json()?).await?;
6598                        AvroValue::from_serialize(&result)
6599                    })
6600                })),
6601            );
6602    }
6603
6604    /// Register a query handler on the lossless fixed Avro Value surface.
6605    pub fn register_query_avro_value<F, Fut>(
6606        &mut self,
6607        workflow_type: impl Into<String>,
6608        query_name: impl Into<String>,
6609        handler: F,
6610    ) where
6611        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6612        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6613    {
6614        self.queries
6615            .entry(workflow_type.into())
6616            .or_default()
6617            .insert(
6618                query_name.into(),
6619                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6620            );
6621    }
6622
6623    /// Register a named query against deterministically replayed instance state.
6624    ///
6625    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
6626    /// same state type `S`. The handler receives an immutable, detached state
6627    /// clone, so successful and failed queries cannot affect workflow execution
6628    /// or the state reconstructed by a later query.
6629    pub fn register_replayed_query<S, F, Fut>(
6630        &mut self,
6631        workflow_type: impl Into<String>,
6632        query_name: impl Into<String>,
6633        handler: F,
6634    ) where
6635        S: Clone + Send + Sync + 'static,
6636        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6637        Fut: Future<Output = Result<Value>> + Send + 'static,
6638    {
6639        let handler = Arc::new(handler);
6640        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6641            let state = state.downcast::<S>().map_err(|_| {
6642                "registered query state type does not match the replayed workflow state".to_string()
6643            })?;
6644            let handler = Arc::clone(&handler);
6645            Ok(Box::pin(async move {
6646                let result = handler(ctx, state, args.into_json()?).await?;
6647                AvroValue::from_serialize(&result)
6648            }))
6649        });
6650
6651        self.queries
6652            .entry(workflow_type.into())
6653            .or_default()
6654            .insert(
6655                query_name.into(),
6656                RegisteredQuery::Replayed {
6657                    state_type: TypeId::of::<S>(),
6658                    handler: erased_handler,
6659                },
6660            );
6661    }
6662
6663    /// Register a replayed-state query on the lossless fixed Avro Value surface.
6664    pub fn register_replayed_query_avro_value<S, F, Fut>(
6665        &mut self,
6666        workflow_type: impl Into<String>,
6667        query_name: impl Into<String>,
6668        handler: F,
6669    ) where
6670        S: Clone + Send + Sync + 'static,
6671        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6672        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6673    {
6674        let handler = Arc::new(handler);
6675        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6676            let state = state.downcast::<S>().map_err(|_| {
6677                "registered query state type does not match the replayed workflow state".to_string()
6678            })?;
6679            Ok(Box::pin(handler(ctx, state, args)))
6680        });
6681
6682        self.queries
6683            .entry(workflow_type.into())
6684            .or_default()
6685            .insert(
6686                query_name.into(),
6687                RegisteredQuery::Replayed {
6688                    state_type: TypeId::of::<S>(),
6689                    handler: erased_handler,
6690                },
6691            );
6692    }
6693
6694    /// Register a synchronous workflow update handler.
6695    pub fn register_update<F, Fut>(
6696        &mut self,
6697        workflow_type: impl Into<String>,
6698        update_name: impl Into<String>,
6699        handler: F,
6700    ) where
6701        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6702        Fut: Future<Output = Result<Value>> + Send + 'static,
6703    {
6704        let handler = Arc::new(handler);
6705        self.updates
6706            .entry(workflow_type.into())
6707            .or_default()
6708            .insert(
6709                update_name.into(),
6710                Arc::new(move |ctx, args| {
6711                    let handler = Arc::clone(&handler);
6712                    Box::pin(async move {
6713                        let result = handler(ctx, args.into_json()?).await?;
6714                        AvroValue::from_serialize(&result)
6715                    })
6716                }),
6717            );
6718    }
6719
6720    /// Register an update handler on the lossless fixed Avro Value surface.
6721    pub fn register_update_avro_value<F, Fut>(
6722        &mut self,
6723        workflow_type: impl Into<String>,
6724        update_name: impl Into<String>,
6725        handler: F,
6726    ) where
6727        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6728        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6729    {
6730        self.updates
6731            .entry(workflow_type.into())
6732            .or_default()
6733            .insert(
6734                update_name.into(),
6735                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6736            );
6737    }
6738
6739    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6740        let mut command_contracts = serde_json::Map::new();
6741        for workflow_type in self.workflows.keys() {
6742            let mut queries = self
6743                .queries
6744                .get(workflow_type)
6745                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6746                .unwrap_or_default();
6747            queries.sort();
6748            let mut updates = self
6749                .updates
6750                .get(workflow_type)
6751                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6752                .unwrap_or_default();
6753            updates.sort();
6754            command_contracts.insert(
6755                workflow_type.clone(),
6756                json!({
6757                    "queries": queries,
6758                    "query_contracts": [],
6759                    "signals": [],
6760                    "signal_contracts": [],
6761                    "updates": updates,
6762                    "update_contracts": [],
6763                    "update_validators": [],
6764                }),
6765            );
6766        }
6767
6768        self.client
6769            .register_worker_with_definition_fingerprints(
6770                &self.worker_id,
6771                &self.task_queue,
6772                self.workflows.keys().cloned().collect(),
6773                self.activities.keys().cloned().collect(),
6774                self.max_concurrent_workflow_tasks,
6775                self.max_concurrent_activity_tasks,
6776                [
6777                    Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6778                    Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6779                    Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6780                    Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6781                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6782                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6783                    worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6784                        .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6785                ]
6786                .into_iter()
6787                .flatten()
6788                .collect(),
6789                Value::Object(command_contracts),
6790                Some(
6791                    self.workflows
6792                        .iter()
6793                        .filter_map(|(workflow_type, workflow)| {
6794                            workflow
6795                                .definition_fingerprint
6796                                .as_ref()
6797                                .map(|fingerprint| (workflow_type.clone(), fingerprint.clone()))
6798                        })
6799                        .collect(),
6800                ),
6801            )
6802            .await
6803    }
6804
6805    /// Run until shutdown or a terminal worker error occurs.
6806    ///
6807    /// Empty long-poll expirations do not stop the worker. Retryable poll and
6808    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
6809    /// authentication, protocol, and other non-retryable failures are returned.
6810    pub async fn run(&self) -> Result<()> {
6811        self.run_until(std::future::pending::<()>()).await
6812    }
6813
6814    /// Run until `shutdown` resolves or a terminal worker error occurs.
6815    ///
6816    /// This has the same liveness and terminal-error contract as [`Worker::run`].
6817    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6818    where
6819        F: Future<Output = ()>,
6820    {
6821        let stop = Arc::new(AtomicBool::new(false));
6822        let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6823        let worker = self.with_storage_admission(Arc::clone(&stop));
6824        let run = worker.run_with_storage_admission(Arc::clone(&stop));
6825        tokio::pin!(run);
6826        tokio::pin!(shutdown);
6827        tokio::select! {
6828            result = &mut run => result,
6829            _ = &mut shutdown => {
6830                stop.store(true, Ordering::SeqCst);
6831                run.await
6832            }
6833        }
6834    }
6835
6836    fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6837        let mut worker = self.clone();
6838        worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6839            policy: self.retry_policy,
6840            stop,
6841        });
6842        worker
6843    }
6844
6845    async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6846        let registration = self.register().await?;
6847        if !registration.registered {
6848            return Err(Error::WorkerLoop(format!(
6849                "worker registration for {:?} was not accepted",
6850                self.worker_id
6851            )));
6852        }
6853        let registered_worker_id = registration.worker_id.clone();
6854        let primary = self.run_registered_until(stop, registration).await;
6855        let deregistration = self
6856            .client
6857            .deregister_worker_registration(&registered_worker_id)
6858            .await;
6859
6860        match (primary, deregistration) {
6861            (Ok(()), Ok(_)) => Ok(()),
6862            (Ok(()), Err(deregistration)) => Err(deregistration),
6863            (Err(primary), Ok(_)) => Err(primary),
6864            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6865                primary: Box::new(primary),
6866                deregistration: Box::new(deregistration),
6867            }),
6868        }
6869    }
6870
6871    async fn run_registered_until(
6872        &self,
6873        stop: Arc<AtomicBool>,
6874        registration: RegisterWorkerResponse,
6875    ) -> Result<()> {
6876        let heartbeat_interval = Duration::from_secs(
6877            registration
6878                .heartbeat_interval_seconds
6879                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6880        );
6881        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
6882        // from the completion of the preceding attempt, including its bounded
6883        // retries. A fixed-epoch interval can leave an already-due tick queued
6884        // while an acknowledgement is slow, producing a catch-up heartbeat as
6885        // soon as that request completes.
6886        let heartbeat = tokio::time::sleep(Duration::ZERO);
6887        tokio::pin!(heartbeat);
6888        // Poll responses may already have leased server-side work by the time
6889        // they become ready, so each poller owns its responses through
6890        // completion or failure instead of racing raw polls in this select.
6891        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6892            let worker = self.clone();
6893            let stop = Arc::clone(&stop);
6894            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6895        });
6896        let mut activity_poller = (!self.activities.is_empty()).then(|| {
6897            let worker = self.clone();
6898            let stop = Arc::clone(&stop);
6899            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6900        });
6901        let mut query_poller = (!self.queries.is_empty()).then(|| {
6902            let worker = self.clone();
6903            let stop = Arc::clone(&stop);
6904            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6905        });
6906
6907        loop {
6908            tokio::select! {
6909                _ = wait_for_worker_stop(&stop) => {
6910                    stop.store(true, Ordering::SeqCst);
6911                    break;
6912                }
6913                _ = &mut heartbeat => {
6914                    let result = self.retry_worker_operation(|| {
6915                        self.client.heartbeat_worker(
6916                            &self.worker_id,
6917                            self.max_concurrent_workflow_tasks,
6918                            self.max_concurrent_activity_tasks,
6919                        )
6920                    }).await;
6921                    heartbeat
6922                        .as_mut()
6923                        .reset(tokio::time::Instant::now() + heartbeat_interval);
6924                    match result {
6925                        Ok(acknowledgement) => {
6926                            if let Some(observer) = &self.heartbeat_observer {
6927                                observer(&WorkerHeartbeatObservation {
6928                                    worker_id: self.worker_id.clone(),
6929                                    task_queue: self.task_queue.clone(),
6930                                    acknowledged_at_unix_millis: SystemTime::now()
6931                                        .duration_since(UNIX_EPOCH)
6932                                        .unwrap_or_default()
6933                                        .as_millis()
6934                                        .min(u64::MAX as u128)
6935                                        as u64,
6936                                    acknowledgement,
6937                                });
6938                            }
6939                        }
6940                        Err(error) => {
6941                            stop.store(true, Ordering::SeqCst);
6942                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6943                            return Err(error);
6944                        }
6945                    }
6946                }
6947                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6948                    workflow_poller = None;
6949                    let stopped_by_server = stop.load(Ordering::SeqCst);
6950                    stop.store(true, Ordering::SeqCst);
6951                    let poller_result = optional_poller_result("workflow", result);
6952                    let join_result =
6953                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6954                    poller_result?;
6955                    join_result?;
6956                    if stopped_by_server {
6957                        return Ok(());
6958                    }
6959                    return Err(Error::WorkerLoop(
6960                        "workflow poller stopped unexpectedly".to_string(),
6961                    ));
6962                }
6963                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6964                    activity_poller = None;
6965                    let stopped_by_server = stop.load(Ordering::SeqCst);
6966                    stop.store(true, Ordering::SeqCst);
6967                    let poller_result = optional_poller_result("activity", result);
6968                    let join_result =
6969                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6970                    poller_result?;
6971                    join_result?;
6972                    if stopped_by_server {
6973                        return Ok(());
6974                    }
6975                    return Err(Error::WorkerLoop(
6976                        "activity poller stopped unexpectedly".to_string(),
6977                    ));
6978                }
6979                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6980                    query_poller = None;
6981                    let stopped_by_server = stop.load(Ordering::SeqCst);
6982                    stop.store(true, Ordering::SeqCst);
6983                    let poller_result = optional_poller_result("query", result);
6984                    let join_result =
6985                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6986                    poller_result?;
6987                    join_result?;
6988                    if stopped_by_server {
6989                        return Ok(());
6990                    }
6991                    return Err(Error::WorkerLoop(
6992                        "query poller stopped unexpectedly".to_string(),
6993                    ));
6994                }
6995            }
6996        }
6997
6998        join_pollers(
6999            workflow_poller.take(),
7000            activity_poller.take(),
7001            query_poller.take(),
7002        )
7003        .await
7004    }
7005
7006    /// Poll and settle at most one task from each enabled task family.
7007    ///
7008    /// A workflow may reach its server-enforced run deadline while this worker
7009    /// holds a task. When the completion endpoint authoritatively rejects that
7010    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
7011    /// terminal `run_status=failed`, the workflow tick is considered settled:
7012    /// the late command was not recorded and cannot replace the terminal run.
7013    /// Every other completion rejection remains an error. This worker-level
7014    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
7015    /// only bounds how long a client waits for a result.
7016    ///
7017    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
7018    /// the original [`Error::Http`] status and response body.
7019    pub async fn run_once(&self) -> Result<usize> {
7020        let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
7021        let mut handled = 0;
7022        match worker.poll_workflow_once().await? {
7023            ManagedPollOutcome::Handled => handled += 1,
7024            ManagedPollOutcome::Stop => return Ok(handled),
7025            ManagedPollOutcome::Idle => {}
7026        }
7027        match worker.poll_activity_once().await? {
7028            ManagedPollOutcome::Handled => handled += 1,
7029            ManagedPollOutcome::Stop => return Ok(handled),
7030            ManagedPollOutcome::Idle => {}
7031        }
7032        if !self.queries.is_empty() {
7033            match worker.poll_query_once().await? {
7034                ManagedPollOutcome::Handled => handled += 1,
7035                ManagedPollOutcome::Stop => return Ok(handled),
7036                ManagedPollOutcome::Idle => {}
7037            }
7038        }
7039        Ok(handled)
7040    }
7041
7042    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
7043        let poll_request_id = unique_request_id("rust-workflow-poll");
7044        let response = self
7045            .retry_worker_operation(|| {
7046                self.client.poll_workflow_task_response_with_request_id(
7047                    &self.worker_id,
7048                    &self.task_queue,
7049                    self.poll_timeout,
7050                    &poll_request_id,
7051                    0,
7052                )
7053            })
7054            .await;
7055        let Some(response) = self.settle_worker_poll_response(response).await? else {
7056            return Ok(ManagedPollOutcome::Idle);
7057        };
7058        if response.outcome().should_stop() {
7059            return Ok(ManagedPollOutcome::Stop);
7060        }
7061        let memo_updates_supported =
7062            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
7063        let Some(task) = response.task else {
7064            return Ok(ManagedPollOutcome::Idle);
7065        };
7066
7067        let task_id = task.task_id.clone();
7068        let attempt = task.workflow_task_attempt;
7069        let run_id = task.run_id.clone();
7070        let lease_owner = task
7071            .lease_owner
7072            .clone()
7073            .unwrap_or_else(|| self.worker_id.clone());
7074
7075        match self.execute_workflow_task_decision(task) {
7076            Ok(decision)
7077                if commands_use_workflow_memo_updates(&decision.commands)
7078                    && !memo_updates_supported =>
7079            {
7080                self.client
7081                    .fail_workflow_task(
7082                        &task_id,
7083                        &lease_owner,
7084                        attempt,
7085                        Error::WorkflowMemoUpdatesUnavailable.to_string(),
7086                    )
7087                    .await?;
7088            }
7089            Ok(decision) if decision.commands.is_empty() => {
7090                // A replay can consume a recorded pending durable command
7091                // without producing a new command. The standalone protocol
7092                // acknowledges that state through the typed waiting outcome;
7093                // an empty completion is rejected by servers that require at
7094                // least one executable command.
7095                self.client
7096                    .fail_workflow_task_with_type(
7097                        &task_id,
7098                        &lease_owner,
7099                        attempt,
7100                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
7101                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
7102                    )
7103                    .await?;
7104            }
7105            Ok(decision) => {
7106                let completion = self
7107                    .client
7108                    .complete_workflow_task_with_message_streams(
7109                        &task_id,
7110                        &lease_owner,
7111                        attempt,
7112                        decision.commands,
7113                        decision.message_stream_cursors,
7114                        decision.message_stream_waits,
7115                    )
7116                    .await;
7117                if let Err(error) = completion {
7118                    if !workflow_task_completion_is_terminal_timeout(
7119                        &error,
7120                        &task_id,
7121                        attempt,
7122                        run_id.as_deref(),
7123                    ) {
7124                        return Err(error);
7125                    }
7126                }
7127            }
7128            Err(error) => {
7129                self.client
7130                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
7131                    .await?;
7132            }
7133        }
7134
7135        Ok(ManagedPollOutcome::Handled)
7136    }
7137
7138    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7139        while !stop.load(Ordering::SeqCst) {
7140            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
7141                stop.store(true, Ordering::SeqCst);
7142                break;
7143            }
7144        }
7145
7146        Ok(())
7147    }
7148
7149    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
7150        let poll_request_id = unique_request_id("rust-activity-poll");
7151        let response = self
7152            .retry_worker_operation(|| {
7153                self.client.poll_activity_task_response_with_request_id(
7154                    &self.worker_id,
7155                    &self.task_queue,
7156                    self.poll_timeout,
7157                    &poll_request_id,
7158                    0,
7159                )
7160            })
7161            .await;
7162        let Some(response) = self.settle_worker_poll_response(response).await? else {
7163            return Ok(ManagedPollOutcome::Idle);
7164        };
7165        if response.outcome().should_stop() {
7166            return Ok(ManagedPollOutcome::Stop);
7167        }
7168        let Some(task) = response.task else {
7169            return Ok(ManagedPollOutcome::Idle);
7170        };
7171
7172        let task_id = task.task_id.clone();
7173        let attempt_id = task
7174            .activity_attempt_id
7175            .clone()
7176            .or(task.attempt_id.clone())
7177            .unwrap_or_default();
7178        let lease_owner = task
7179            .lease_owner
7180            .clone()
7181            .unwrap_or_else(|| self.worker_id.clone());
7182        let codec = task.payload_codec.clone();
7183        let result = self.execute_activity_task(task).await;
7184        match result {
7185            Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
7186            Ok(value) => {
7187                let completion = self
7188                    .client
7189                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
7190                    .await;
7191                if let Err(error) = completion {
7192                    if !activity_task_rejection_is_final(&error) {
7193                        return Err(error);
7194                    }
7195                }
7196            }
7197            Err(error) => {
7198                let failure = self
7199                    .client
7200                    .fail_activity_task(
7201                        &task_id,
7202                        &attempt_id,
7203                        &lease_owner,
7204                        error.to_string(),
7205                        false,
7206                    )
7207                    .await;
7208                if let Err(error) = failure {
7209                    if !activity_task_rejection_is_final(&error) {
7210                        return Err(error);
7211                    }
7212                }
7213            }
7214        }
7215
7216        Ok(ManagedPollOutcome::Handled)
7217    }
7218
7219    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7220        while !stop.load(Ordering::SeqCst) {
7221            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
7222                stop.store(true, Ordering::SeqCst);
7223                break;
7224            }
7225        }
7226
7227        Ok(())
7228    }
7229
7230    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
7231        let poll_request_id = unique_request_id("rust-query-poll");
7232        let response = self
7233            .retry_worker_operation(|| {
7234                self.client.poll_query_task_response_with_request_id(
7235                    &self.worker_id,
7236                    &self.task_queue,
7237                    self.poll_timeout,
7238                    &poll_request_id,
7239                    0,
7240                )
7241            })
7242            .await;
7243        let Some(response) = self.settle_worker_poll_response(response).await? else {
7244            return Ok(ManagedPollOutcome::Idle);
7245        };
7246        if response.outcome().should_stop() {
7247            return Ok(ManagedPollOutcome::Stop);
7248        }
7249        let Some(task) = response.task else {
7250            return Ok(ManagedPollOutcome::Idle);
7251        };
7252
7253        let query_task_id = task.query_task_id.clone();
7254        let attempt = task.query_task_attempt;
7255        let lease_owner = task
7256            .lease_owner
7257            .clone()
7258            .unwrap_or_else(|| self.worker_id.clone());
7259        let codec = task.payload_codec.clone();
7260
7261        match self.execute_query_task(task).await {
7262            Ok(value) => {
7263                let result_envelope = match encode_typed_envelope(&value, &codec) {
7264                    Ok(result_envelope) => result_envelope,
7265                    Err(error) => {
7266                        let failure = self
7267                            .client
7268                            .fail_query_task(
7269                                &query_task_id,
7270                                &lease_owner,
7271                                attempt,
7272                                error.to_string(),
7273                                "query_result_encode_failed",
7274                                "QueryResultEncodeFailed",
7275                            )
7276                            .await;
7277                        if let Err(error) = failure {
7278                            if !query_task_rejection_is_final(&error) {
7279                                return Err(error);
7280                            }
7281                        }
7282                        return Ok(ManagedPollOutcome::Handled);
7283                    }
7284                };
7285
7286                if let Err(error) = self
7287                    .client
7288                    .complete_query_task_with_envelope(
7289                        &query_task_id,
7290                        &lease_owner,
7291                        attempt,
7292                        value.clone().into_json()?,
7293                        result_envelope,
7294                    )
7295                    .await
7296                {
7297                    if !query_task_rejection_is_final(&error) {
7298                        return Err(error);
7299                    }
7300                }
7301            }
7302            Err(failure) => {
7303                let result = self
7304                    .client
7305                    .fail_query_task(
7306                        &query_task_id,
7307                        &lease_owner,
7308                        attempt,
7309                        failure.message,
7310                        failure.reason,
7311                        failure.failure_type,
7312                    )
7313                    .await;
7314                if let Err(error) = result {
7315                    if !query_task_rejection_is_final(&error) {
7316                        return Err(error);
7317                    }
7318                }
7319            }
7320        }
7321
7322        Ok(ManagedPollOutcome::Handled)
7323    }
7324
7325    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7326        while !stop.load(Ordering::SeqCst) {
7327            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
7328                stop.store(true, Ordering::SeqCst);
7329                break;
7330            }
7331        }
7332
7333        Ok(())
7334    }
7335
7336    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7337    where
7338        F: FnMut() -> Fut,
7339        Fut: Future<Output = Result<T>>,
7340    {
7341        let mut retries = 0;
7342
7343        loop {
7344            match operation().await {
7345                Err(error)
7346                    if worker_operation_is_retryable(&error)
7347                        && retries < self.retry_policy.max_retries =>
7348                {
7349                    retries += 1;
7350                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7351                }
7352                result => return result,
7353            }
7354        }
7355    }
7356
7357    async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7358        match response {
7359            Ok(response) => Ok(Some(response)),
7360            Err(error) => {
7361                let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7362                    return Err(error);
7363                };
7364                let minimum_delay = self
7365                    .retry_policy
7366                    .initial_backoff
7367                    .max(Duration::from_millis(1));
7368                let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7369                tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7370                Ok(None)
7371            }
7372        }
7373    }
7374
7375    async fn execute_query_task(
7376        &self,
7377        mut task: QueryTask,
7378    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7379        validate_query_task_payloads(&task).map_err(|error| {
7380            QueryTaskExecutionFailure::new(
7381                "query_payload_decode_failed",
7382                error.to_string(),
7383                "QueryPayloadDecodeFailed",
7384            )
7385        })?;
7386
7387        if !self.workflows.contains_key(&task.workflow_type) {
7388            return Err(QueryTaskExecutionFailure::new(
7389                "query_workflow_type_not_registered",
7390                format!("no workflow registered for type {:?}", task.workflow_type),
7391                "WorkflowTypeNotRegistered",
7392            ));
7393        }
7394
7395        let Some(handlers) = self.queries.get(&task.workflow_type) else {
7396            return Err(QueryTaskExecutionFailure::new(
7397                "query_handler_unavailable",
7398                format!(
7399                    "query handlers are unavailable for workflow type {:?}",
7400                    task.workflow_type
7401                ),
7402                "QueryHandlerUnavailable",
7403            ));
7404        };
7405        let Some(query) = handlers.get(&task.query_name) else {
7406            return Err(QueryTaskExecutionFailure::new(
7407                "rejected_unknown_query",
7408                format!("unknown query {:?}", task.query_name),
7409                "QueryFailed",
7410            ));
7411        };
7412
7413        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7414            .map_err(|error| {
7415            QueryTaskExecutionFailure::new(
7416                "query_payload_decode_failed",
7417                format!("cannot decode query arguments: {error}"),
7418                "QueryPayloadDecodeFailed",
7419            )
7420        })?;
7421        let workflow_input_typed =
7422            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7423                .map_err(|error| {
7424                    QueryTaskExecutionFailure::new(
7425                        "query_workflow_state_unavailable",
7426                        format!("cannot decode workflow start input: {error}"),
7427                        "QueryWorkflowStateUnavailable",
7428                    )
7429                })?;
7430        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7431            QueryTaskExecutionFailure::new(
7432                "query_workflow_state_unavailable",
7433                format!("cannot project workflow start input: {error}"),
7434                "QueryWorkflowStateUnavailable",
7435            )
7436        })?;
7437        hydrate_query_history_from_export(&mut task).map_err(|error| {
7438            QueryTaskExecutionFailure::new(
7439                "query_workflow_state_unavailable",
7440                format!("cannot restore query history snapshot: {error}"),
7441                "QueryWorkflowStateUnavailable",
7442            )
7443        })?;
7444        enrich_query_history_from_export(&mut task).map_err(|error| {
7445            QueryTaskExecutionFailure::new(
7446                "query_workflow_state_unavailable",
7447                format!("cannot restore compact query history payloads: {error}"),
7448                "QueryWorkflowStateUnavailable",
7449            )
7450        })?;
7451        let signal_events = query_signal_events(&task).map_err(|error| {
7452            QueryTaskExecutionFailure::new(
7453                "query_workflow_state_unavailable",
7454                format!("cannot decode committed workflow signals: {error}"),
7455                "QueryWorkflowStateUnavailable",
7456            )
7457        })?;
7458        let history_events = Arc::new(std::mem::take(&mut task.history_events));
7459        let context = QueryContext {
7460            workflow_id: task.workflow_id,
7461            run_id: task.run_id,
7462            workflow_type: task.workflow_type.clone(),
7463            run_status: task.run_status,
7464            workflow_input,
7465            workflow_input_avro_value: workflow_input_typed.clone(),
7466            history_events: Arc::clone(&history_events),
7467            signal_events: Arc::new(signal_events),
7468        };
7469
7470        let future = match query {
7471            RegisteredQuery::Snapshot(handler) => handler(context, args),
7472            RegisteredQuery::Replayed {
7473                state_type,
7474                handler,
7475            } => {
7476                let workflow = self
7477                    .workflows
7478                    .get(&task.workflow_type)
7479                    .expect("workflow registration was checked above");
7480                if workflow.state_type != Some(*state_type) {
7481                    return Err(QueryTaskExecutionFailure::new(
7482                        "query_workflow_state_unavailable",
7483                        "replayed query state type does not match its workflow registration",
7484                        "QueryWorkflowStateUnavailable",
7485                    ));
7486                }
7487                let replay = workflow.replay.as_ref().ok_or_else(|| {
7488                    QueryTaskExecutionFailure::new(
7489                        "query_workflow_state_unavailable",
7490                        format!(
7491                            "workflow type {:?} is not registered for instance-state replay",
7492                            task.workflow_type
7493                        ),
7494                        "QueryWorkflowStateUnavailable",
7495                    )
7496                })?;
7497                let workflow_state = Arc::new(Mutex::new(
7498                    WorkflowState::new_with_identity(
7499                        history_events.as_ref().clone(),
7500                        context.workflow_id.clone(),
7501                        context.run_id.clone(),
7502                        self.task_queue.clone(),
7503                        task.payload_codec,
7504                        None,
7505                    )
7506                    .map_err(|error| {
7507                        QueryTaskExecutionFailure::new(
7508                            "query_workflow_state_unavailable",
7509                            format!("workflow replay failed before query: {error}"),
7510                            "QueryWorkflowStateUnavailable",
7511                        )
7512                    })?,
7513                ));
7514                let workflow_context = WorkflowContext {
7515                    state: workflow_state,
7516                };
7517                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7518                let mut cx = TaskContext::from_waker(noop_waker_ref());
7519                match invocation.future.as_mut().poll(&mut cx) {
7520                    Poll::Ready(Ok(_)) => {
7521                        workflow_context
7522                            .ensure_history_consumed()
7523                            .map_err(|error| {
7524                                QueryTaskExecutionFailure::new(
7525                                    "query_workflow_state_unavailable",
7526                                    format!("workflow replay failed before query: {error}"),
7527                                    "QueryWorkflowStateUnavailable",
7528                                )
7529                            })?;
7530                    }
7531                    Poll::Ready(Err(error)) => {
7532                        return Err(QueryTaskExecutionFailure::new(
7533                            "query_workflow_state_unavailable",
7534                            format!("workflow replay failed before query: {error}"),
7535                            "QueryWorkflowStateUnavailable",
7536                        ));
7537                    }
7538                    Poll::Pending => {
7539                        let commands = workflow_context.take_commands().map_err(|error| {
7540                            QueryTaskExecutionFailure::new(
7541                                "query_workflow_state_unavailable",
7542                                format!("workflow replay failed before query: {error}"),
7543                                "QueryWorkflowStateUnavailable",
7544                            )
7545                        })?;
7546                        if commands.is_empty()
7547                            && !workflow_context
7548                                .matched_recorded_pending()
7549                                .map_err(|error| {
7550                                    QueryTaskExecutionFailure::new(
7551                                        "query_workflow_state_unavailable",
7552                                        format!("workflow replay failed before query: {error}"),
7553                                        "QueryWorkflowStateUnavailable",
7554                                    )
7555                                })?
7556                        {
7557                            return Err(QueryTaskExecutionFailure::new(
7558                                "query_workflow_state_unavailable",
7559                                "workflow replay yielded without a durable command",
7560                                "QueryWorkflowStateUnavailable",
7561                            ));
7562                        }
7563                    }
7564                }
7565                let state = (invocation.snapshot)().map_err(|error| {
7566                    QueryTaskExecutionFailure::new(
7567                        "query_workflow_state_unavailable",
7568                        format!("cannot snapshot replayed workflow state: {error}"),
7569                        "QueryWorkflowStateUnavailable",
7570                    )
7571                })?;
7572                handler(context, state, args).map_err(|message| {
7573                    QueryTaskExecutionFailure::new(
7574                        "query_workflow_state_unavailable",
7575                        message,
7576                        "QueryWorkflowStateUnavailable",
7577                    )
7578                })?
7579            }
7580        };
7581
7582        future.await.map_err(|error| {
7583            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7584        })
7585    }
7586
7587    #[cfg(test)]
7588    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7589        Ok(self.execute_workflow_task_decision(task)?.commands)
7590    }
7591
7592    fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7593        validate_workflow_task_payloads(&task)?;
7594
7595        if let Some(update_id) = task
7596            .workflow_update_id
7597            .as_deref()
7598            .filter(|update_id| !update_id.is_empty())
7599        {
7600            return self
7601                .execute_update_task(&task, update_id)
7602                .map(WorkflowTaskDecision::without_message_streams);
7603        }
7604
7605        let workflow = self
7606            .workflows
7607            .get(&task.workflow_type)
7608            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7609        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7610        let resume_signal = decode_resume_signal(&task)?;
7611        let history_budget = WorkflowHistoryBudget {
7612            event_count: task
7613                .total_history_events
7614                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7615            size_bytes: task.history_size_bytes,
7616            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7617            pressure: task.history_budget_pressure.clone(),
7618        };
7619        let workflow_command_identity = task
7620            .workflow_command_id
7621            .clone()
7622            .filter(|identity| !identity.is_empty())
7623            .unwrap_or_default();
7624        let mut workflow_state = WorkflowState::new_with_identity(
7625            task.history_events,
7626            task.workflow_id,
7627            task.run_id,
7628            self.task_queue.clone(),
7629            task.payload_codec.clone(),
7630            resume_signal,
7631        )?;
7632        workflow_state.history_budget = history_budget;
7633        workflow_state.workflow_command_identity = workflow_command_identity;
7634        workflow_state.cancel_requested = task.cancel_requested;
7635        let state = Arc::new(Mutex::new(workflow_state));
7636        let ctx = WorkflowContext { state };
7637        let mut future = (workflow.execute)(ctx.clone(), input);
7638        let mut cx = TaskContext::from_waker(noop_waker_ref());
7639
7640        match future.as_mut().poll(&mut cx) {
7641            Poll::Ready(Ok(result)) => {
7642                ctx.ensure_history_consumed()?;
7643                let result = encode_typed_envelope(&result, &task.payload_codec)?;
7644                let mut commands = ctx.take_commands()?;
7645                commands.push(json!({
7646                    "type": "complete_workflow",
7647                    "result": result
7648                }));
7649                self.message_stream_decision(&ctx, commands)
7650            }
7651            Poll::Ready(Err(error)) => {
7652                if let Error::ContinueAsNew(request) = error {
7653                    let mut commands = ctx.take_commands()?;
7654                    if let Some(command) = ctx.continue_as_new_command(request)? {
7655                        commands.push(command);
7656                    }
7657                    ctx.ensure_history_consumed()?;
7658                    return self.message_stream_decision(&ctx, commands);
7659                }
7660                if workflow_task_integrity_error(&error) {
7661                    // Replay and protocol failures describe the workflow-task
7662                    // decision itself. Preserve their specific failure reason
7663                    // instead of replacing it with the derivative fact that
7664                    // recorded commands remain unconsumed.
7665                    return Err(error);
7666                }
7667                // A handler error must not hide a committed durable command that
7668                // upgraded workflow code no longer consumes.
7669                ctx.ensure_history_consumed()?;
7670                let mut commands = ctx.take_commands()?;
7671                commands.push(workflow_failure_command(
7672                    &error,
7673                    recorded_activity_failure_boundary(&ctx, &error),
7674                ));
7675                self.message_stream_decision(&ctx, commands)
7676            }
7677            Poll::Pending => {
7678                let commands = ctx.take_commands()?;
7679                if commands.is_empty() && !ctx.matched_recorded_pending()? {
7680                    Err(Error::WorkflowYieldedWithoutCommand)
7681                } else {
7682                    self.message_stream_decision(&ctx, commands)
7683                }
7684            }
7685        }
7686    }
7687
7688    fn message_stream_decision(
7689        &self,
7690        ctx: &WorkflowContext,
7691        commands: Vec<Value>,
7692    ) -> Result<WorkflowTaskDecision> {
7693        let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7694        Ok(WorkflowTaskDecision {
7695            commands,
7696            message_stream_cursors,
7697            message_stream_waits,
7698        })
7699    }
7700
7701    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7702        if !self.workflows.contains_key(&task.workflow_type) {
7703            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7704        }
7705
7706        let accepted = task.history_events.iter().rev().find_map(|event| {
7707            (event.event_type == "UpdateAccepted"
7708                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7709            .then_some(&event.payload)
7710        });
7711        let update_name = accepted
7712            .and_then(|payload| payload.get("update_name"))
7713            .and_then(Value::as_str)
7714            .or(task.update_name.as_deref())
7715            .unwrap_or_default();
7716        let Some(handler) = self
7717            .updates
7718            .get(&task.workflow_type)
7719            .and_then(|handlers| handlers.get(update_name))
7720        else {
7721            return Ok(vec![json!({
7722                "type": "fail_update",
7723                "update_id": update_id,
7724                "message": format!(
7725                    "no update handler is registered for {}.{update_name}",
7726                    task.workflow_type
7727                ),
7728                "exception_type": "UnknownUpdate",
7729                "non_retryable": true,
7730            })]);
7731        };
7732        let arguments = accepted
7733            .and_then(|payload| payload.get("arguments"))
7734            .or(task.arguments.as_ref());
7735        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7736        let context = QueryContext {
7737            workflow_id: task.workflow_id.clone(),
7738            run_id: task.run_id.clone(),
7739            workflow_type: task.workflow_type.clone(),
7740            run_status: Some("running".to_string()),
7741            workflow_input: Value::Null,
7742            workflow_input_avro_value: AvroValue::Null,
7743            history_events: Arc::new(task.history_events.clone()),
7744            signal_events: Arc::new(Vec::new()),
7745        };
7746        let mut future = handler(context, arguments);
7747        let mut cx = TaskContext::from_waker(noop_waker_ref());
7748
7749        match future.as_mut().poll(&mut cx) {
7750            Poll::Ready(Ok(result)) => Ok(vec![json!({
7751                "type": "complete_update",
7752                "update_id": update_id,
7753                "result": encode_typed_envelope(&result, &task.payload_codec)?,
7754            })]),
7755            Poll::Ready(Err(error)) => Ok(vec![json!({
7756                "type": "fail_update",
7757                "update_id": update_id,
7758                "message": error.to_string(),
7759                "exception_type": "UpdateFailed",
7760                "non_retryable": true,
7761            })]),
7762            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7763        }
7764    }
7765
7766    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7767        validate_activity_task_payloads(&task)?;
7768
7769        let handler = self
7770            .activities
7771            .get(&task.activity_type)
7772            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7773        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7774        let attempt_id = task
7775            .activity_attempt_id
7776            .clone()
7777            .or(task.attempt_id.clone())
7778            .unwrap_or_default();
7779        let lease_owner = task
7780            .lease_owner
7781            .clone()
7782            .unwrap_or_else(|| self.worker_id.clone());
7783        let ctx = ActivityContext {
7784            client: self.client.clone(),
7785            task_id: task.task_id,
7786            activity_attempt_id: attempt_id,
7787            lease_owner,
7788            activity_type: task.activity_type,
7789            attempt_number: task.attempt_number,
7790            task_queue: self.task_queue.clone(),
7791            worker_id: self.worker_id.clone(),
7792        };
7793
7794        handler(ctx, args).await
7795    }
7796}
7797
7798fn poller_result(
7799    kind: &str,
7800    result: std::result::Result<Result<()>, tokio::task::JoinError>,
7801) -> Result<()> {
7802    match result {
7803        Ok(result) => result,
7804        Err(error) => Err(Error::WorkerLoop(format!(
7805            "{kind} poller join error: {error}"
7806        ))),
7807    }
7808}
7809
7810fn optional_poller_result(
7811    kind: &str,
7812    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7813) -> Result<()> {
7814    match result {
7815        Some(result) => poller_result(kind, result),
7816        None => Ok(()),
7817    }
7818}
7819
7820async fn join_pollers(
7821    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7822    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7823    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7824) -> Result<()> {
7825    let mut first_error = None;
7826
7827    if let Some(handle) = workflow_poller {
7828        if let Err(error) = poller_result("workflow", handle.await) {
7829            first_error.get_or_insert(error);
7830        }
7831    }
7832
7833    if let Some(handle) = activity_poller {
7834        if let Err(error) = poller_result("activity", handle.await) {
7835            first_error.get_or_insert(error);
7836        }
7837    }
7838
7839    if let Some(handle) = query_poller {
7840        if let Err(error) = poller_result("query", handle.await) {
7841            first_error.get_or_insert(error);
7842        }
7843    }
7844
7845    if let Some(error) = first_error {
7846        Err(error)
7847    } else {
7848        Ok(())
7849    }
7850}
7851
7852fn default_worker_id() -> String {
7853    let millis = SystemTime::now()
7854        .duration_since(UNIX_EPOCH)
7855        .unwrap_or_default()
7856        .as_millis();
7857    format!("rust-worker-{}-{millis}", std::process::id())
7858}
7859
7860fn percent_encode_path_segment(segment: &str) -> String {
7861    const HEX: &[u8; 16] = b"0123456789ABCDEF";
7862    let mut encoded = String::with_capacity(segment.len());
7863
7864    for byte in segment.bytes() {
7865        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7866            encoded.push(char::from(byte));
7867        } else {
7868            encoded.push('%');
7869            encoded.push(char::from(HEX[(byte >> 4) as usize]));
7870            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7871        }
7872    }
7873
7874    encoded
7875}
7876
7877fn unique_request_id(prefix: &str) -> String {
7878    let nanos = SystemTime::now()
7879        .duration_since(UNIX_EPOCH)
7880        .unwrap_or_default()
7881        .as_nanos();
7882    format!("{prefix}-{}-{nanos}", std::process::id())
7883}
7884
7885#[derive(Debug)]
7886struct QueryTaskExecutionFailure {
7887    reason: String,
7888    message: String,
7889    failure_type: String,
7890}
7891
7892impl QueryTaskExecutionFailure {
7893    fn new(
7894        reason: impl Into<String>,
7895        message: impl Into<String>,
7896        failure_type: impl Into<String>,
7897    ) -> Self {
7898        Self {
7899            reason: reason.into(),
7900            message: message.into(),
7901            failure_type: failure_type.into(),
7902        }
7903    }
7904}
7905
7906/// Typed local state owned by one deterministic workflow invocation.
7907///
7908/// Use [`WorkflowInstance::update`] for the same state transitions during
7909/// ordinary execution and replay. A replayed query receives a detached
7910/// immutable `Arc<S>` rather than this mutation-capable handle.
7911#[derive(Clone, Debug)]
7912pub struct WorkflowInstance<S> {
7913    state: Arc<Mutex<S>>,
7914}
7915
7916impl<S> WorkflowInstance<S> {
7917    fn new(state: S) -> Self {
7918        Self {
7919            state: Arc::new(Mutex::new(state)),
7920        }
7921    }
7922
7923    /// Read the current workflow-instance state without changing it.
7924    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7925        let state = self
7926            .state
7927            .lock()
7928            .map_err(|_| Error::WorkflowStatePoisoned)?;
7929        Ok(reader(&state))
7930    }
7931
7932    /// Apply one deterministic workflow-instance state transition.
7933    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7934        let mut state = self
7935            .state
7936            .lock()
7937            .map_err(|_| Error::WorkflowStatePoisoned)?;
7938        Ok(transition(&mut state))
7939    }
7940}
7941
7942impl<S: Clone> WorkflowInstance<S> {
7943    fn snapshot(&self) -> Result<S> {
7944        self.read(Clone::clone)
7945    }
7946}
7947
7948#[derive(Clone, Debug, PartialEq)]
7949pub struct MessageStreamMessage {
7950    pub stream_name: String,
7951    pub message_id: String,
7952    pub position: u64,
7953    pub arguments: Vec<AvroValue>,
7954}
7955
7956#[derive(Clone, Debug)]
7957pub struct MessageStream {
7958    ctx: WorkflowContext,
7959    name: String,
7960}
7961
7962impl MessageStream {
7963    /// Wait for one message, then return a bounded currently-available batch.
7964    pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7965        if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7966            return Err(Error::Codec(format!(
7967                "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7968            )));
7969        }
7970        loop {
7971            if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7972                return Ok(batch);
7973            }
7974
7975            self.ctx.record_message_stream_wait(&self.name)?;
7976            let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7977            let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7978            self.ctx.buffer_message_stream_delivery(arguments)?;
7979            if let Some(sequence) = replay_wait_sequence {
7980                self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7981            }
7982        }
7983    }
7984
7985    pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7986        self.receive(1)
7987            .await?
7988            .into_iter()
7989            .next()
7990            .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7991    }
7992}
7993
7994#[derive(Clone, Debug)]
7995pub struct WorkflowContext {
7996    state: Arc<Mutex<WorkflowState>>,
7997}
7998
7999fn valid_memo_key(key: &str) -> bool {
8000    let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
8001
8002    !key.is_empty()
8003        && key.len() <= 64
8004        && (numeric_candidate.is_empty()
8005            || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
8006        && key
8007            .bytes()
8008            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
8009}
8010
8011fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
8012    BASE64
8013        .decode(encode_avro_value(value)?.blob)
8014        .map(|bytes| bytes.len())
8015        .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
8016}
8017
8018fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
8019    let AvroValue::Map(entries) = value else {
8020        return Err(Error::InvalidMemoUpdate(
8021            "entries must serialize to an Avro string-keyed map".to_string(),
8022        ));
8023    };
8024    if require_entries && entries.is_empty() {
8025        return Err(Error::InvalidMemoUpdate(
8026            "at least one entry is required".to_string(),
8027        ));
8028    }
8029    if entries.len() > MAX_MEMO_ENTRIES {
8030        return Err(Error::InvalidMemoUpdate(format!(
8031            "at most {MAX_MEMO_ENTRIES} entries are allowed"
8032        )));
8033    }
8034
8035    for (key, value) in &entries {
8036        if !valid_memo_key(&key) {
8037            return Err(Error::InvalidMemoUpdate(
8038                "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
8039            ));
8040        }
8041        if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
8042            return Err(Error::InvalidMemoUpdate(format!(
8043                "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
8044            )));
8045        }
8046    }
8047
8048    let value = AvroValue::Map(entries);
8049    if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
8050        return Err(Error::InvalidMemoUpdate(format!(
8051            "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
8052        )));
8053    }
8054    Ok(value)
8055}
8056
8057fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
8058    let object = envelope.as_object().ok_or_else(|| {
8059        Error::InvalidMemoUpdate(
8060            "history field must use the public {codec, blob} payload envelope".to_string(),
8061        )
8062    })?;
8063    if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
8064        return Err(Error::InvalidMemoUpdate(
8065            "history field must use exactly the public {codec, blob} payload envelope".to_string(),
8066        ));
8067    }
8068
8069    canonical_memo_entries(
8070        decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
8071        require_entries,
8072    )
8073}
8074
8075impl WorkflowContext {
8076    pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
8077        let name = name.into();
8078        if name.is_empty()
8079            || name.len() > 128
8080            || !name.bytes().all(|byte| {
8081                byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
8082            })
8083        {
8084            return Err(Error::Codec(
8085                "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
8086                    .to_string(),
8087            ));
8088        }
8089        Ok(MessageStream {
8090            ctx: self.clone(),
8091            name,
8092        })
8093    }
8094
8095    fn record_message_stream_wait(&self, name: &str) -> Result<()> {
8096        let mut state = self
8097            .state
8098            .lock()
8099            .map_err(|_| Error::WorkflowStatePoisoned)?;
8100        let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
8101        state
8102            .message_stream_waits
8103            .insert(name.to_string(), position);
8104        Ok(())
8105    }
8106
8107    fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
8108        let mut state = self
8109            .state
8110            .lock()
8111            .map_err(|_| Error::WorkflowStatePoisoned)?;
8112        let cursor = state
8113            .message_stream_cursors
8114            .get(&message.stream_name)
8115            .copied()
8116            .unwrap_or(0);
8117        if message.position <= cursor {
8118            return Ok(());
8119        }
8120        let pending = state
8121            .message_stream_messages
8122            .entry(message.stream_name.clone())
8123            .or_default();
8124        if pending.iter().any(|candidate| {
8125            candidate.position == message.position || candidate.message_id == message.message_id
8126        }) {
8127            return Ok(());
8128        }
8129        pending.push(message);
8130        pending.sort_by_key(|candidate| candidate.position);
8131        Ok(())
8132    }
8133
8134    fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
8135        if let Some(delivery) = decode_message_stream_delivery(arguments)? {
8136            match delivery {
8137                MessageStreamDelivery::Message(message) => {
8138                    let stream_name = message.stream_name.clone();
8139                    self.buffer_message_stream(message)?;
8140                    return Ok(Some(stream_name));
8141                }
8142                MessageStreamDelivery::Cursor {
8143                    stream_name,
8144                    through_position,
8145                } => self.apply_message_stream_cursor(&stream_name, through_position)?,
8146            }
8147        }
8148        Ok(None)
8149    }
8150
8151    fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
8152        let state = self
8153            .state
8154            .lock()
8155            .map_err(|_| Error::WorkflowStatePoisoned)?;
8156        Ok(match state.recorded_commands.get(state.command_cursor) {
8157            Some(RecordedCommand::SignalWait {
8158                sequence,
8159                signal_name,
8160                ..
8161            }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
8162            _ => None,
8163        })
8164    }
8165
8166    fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
8167        let (history, payload_codec) = {
8168            let state = self
8169                .state
8170                .lock()
8171                .map_err(|_| Error::WorkflowStatePoisoned)?;
8172            (
8173                Arc::clone(&state.history_events),
8174                state.payload_codec.clone(),
8175            )
8176        };
8177
8178        let Some(opened_index) = history.iter().position(|event| {
8179            event.event_type == "SignalWaitOpened"
8180                && durable_event_sequence(event) == Some(wait_sequence)
8181                && event.payload.get("signal_name").and_then(Value::as_str)
8182                    == Some(MESSAGE_STREAM_SIGNAL)
8183        }) else {
8184            return Ok(());
8185        };
8186        let boundary_index = history
8187            .iter()
8188            .enumerate()
8189            .skip(opened_index + 1)
8190            .find_map(|(index, event)| {
8191                (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
8192                    && is_authored_command_open_event(event))
8193                .then_some(index)
8194            })
8195            .unwrap_or(history.len());
8196
8197        for event in history[opened_index + 1..boundary_index]
8198            .iter()
8199            .filter(|event| {
8200                event.event_type == "SignalReceived"
8201                    && event.payload.get("signal_name").and_then(Value::as_str)
8202                        == Some(MESSAGE_STREAM_SIGNAL)
8203            })
8204        {
8205            let arguments = decode_signal_event_arguments(event, &payload_codec)?
8206                .into_iter()
8207                .map(AvroValue::into_json)
8208                .collect::<Result<Vec<_>>>()?;
8209            self.buffer_message_stream_delivery(arguments)?;
8210        }
8211        Ok(())
8212    }
8213
8214    fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
8215        let mut state = self
8216            .state
8217            .lock()
8218            .map_err(|_| Error::WorkflowStatePoisoned)?;
8219        let cursor = state
8220            .message_stream_cursors
8221            .entry(name.to_string())
8222            .or_default();
8223        *cursor = (*cursor).max(through_position);
8224        if let Some(pending) = state.message_stream_messages.get_mut(name) {
8225            pending.retain(|message| message.position > through_position);
8226        }
8227        Ok(())
8228    }
8229
8230    fn take_message_stream_batch(
8231        &self,
8232        name: &str,
8233        max_items: usize,
8234    ) -> Result<Option<Vec<MessageStreamMessage>>> {
8235        let mut state = self
8236            .state
8237            .lock()
8238            .map_err(|_| Error::WorkflowStatePoisoned)?;
8239        let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
8240        let pending = state
8241            .message_stream_messages
8242            .entry(name.to_string())
8243            .or_default();
8244        let count = contiguous_message_stream_count(pending, cursor, max_items);
8245        if count == 0 {
8246            return Ok(None);
8247        }
8248        let batch = pending.drain(..count).collect::<Vec<_>>();
8249        let position = batch.last().map(|message| message.position).unwrap_or(0);
8250        state
8251            .message_stream_cursors
8252            .insert(name.to_string(), position);
8253        state.message_stream_waits.remove(name);
8254        Ok(Some(batch))
8255    }
8256
8257    fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
8258        let state = self
8259            .state
8260            .lock()
8261            .map_err(|_| Error::WorkflowStatePoisoned)?;
8262        let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
8263        cursors.sort_by_key(|(name, _)| *name);
8264        let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
8265        waits.sort_by_key(|(name, _)| *name);
8266        Ok((
8267            cursors
8268                .into_iter()
8269                .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
8270                .collect(),
8271            waits
8272                .into_iter()
8273                .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
8274                .collect(),
8275        ))
8276    }
8277    /// Identity of the parent workflow currently being replayed.
8278    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
8279        let state = self
8280            .state
8281            .lock()
8282            .map_err(|_| Error::WorkflowStatePoisoned)?;
8283        Ok(WorkflowIdentity {
8284            workflow_id: state.workflow_id.clone(),
8285            run_id: state.run_id.clone(),
8286        })
8287    }
8288
8289    /// Return the server-published history budget for this workflow task.
8290    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
8291        let state = self
8292            .state
8293            .lock()
8294            .map_err(|_| Error::WorkflowStatePoisoned)?;
8295        Ok(state.history_budget.clone())
8296    }
8297
8298    /// Continue this workflow instance as a fresh run with replacement arguments.
8299    ///
8300    /// Return this value directly from the workflow handler. The worker converts
8301    /// it to the terminal protocol command only after replay has consumed every
8302    /// recorded durable command.
8303    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
8304        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
8305    }
8306
8307    /// Continue as new with optional workflow-type and task-queue overrides.
8308    pub fn continue_as_new_with_options<T: Serialize>(
8309        &self,
8310        options: ContinueAsNewOptions,
8311        args: T,
8312    ) -> Result<Value> {
8313        options.validate()?;
8314        Err(Error::ContinueAsNew(ContinueAsNewRequest {
8315            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
8316            options,
8317        }))
8318    }
8319
8320    pub fn activity<T: Serialize>(
8321        &self,
8322        activity_type: impl Into<String>,
8323        args: T,
8324    ) -> ActivityCall {
8325        self.activity_with_options(activity_type, ActivityOptions::new(), args)
8326    }
8327
8328    pub fn activity_on_queue<T, Q>(
8329        &self,
8330        activity_type: impl Into<String>,
8331        task_queue: Option<Q>,
8332        args: T,
8333    ) -> ActivityCall
8334    where
8335        T: Serialize,
8336        Q: Into<String>,
8337    {
8338        let mut options = ActivityOptions::new();
8339        options.task_queue = task_queue.map(Into::into);
8340        self.activity_with_options(activity_type, options, args)
8341    }
8342
8343    /// Schedule one durable activity with retry, routing, and timeout options.
8344    ///
8345    /// Options are validated before the command is emitted. Once the command is
8346    /// recorded, replay consumes the same activity lifecycle at this command
8347    /// position and never emits a duplicate schedule.
8348    ///
8349    /// ```no_run
8350    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
8351    /// # use std::time::Duration;
8352    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
8353    /// let result = ctx
8354    ///     .activity_with_options(
8355    ///         "charge-card",
8356    ///         ActivityOptions::new()
8357    ///             .task_queue("payments")
8358    ///             .retry_policy(
8359    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
8360    ///                     Duration::from_secs(1),
8361    ///                     2,
8362    ///                     Some(Duration::from_secs(30)),
8363    ///                 ),
8364    ///             )
8365    ///             .start_to_close_timeout(Duration::from_secs(60))
8366    ///             .schedule_to_close_timeout(Duration::from_secs(180))
8367    ///             .heartbeat_timeout(Duration::from_secs(15)),
8368    ///         json!([{"order_id": "order-42"}]),
8369    ///     )
8370    ///     .await;
8371    /// match result {
8372    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
8373    ///         "reason": failure.reason,
8374    ///         "timeout_kind": failure.timeout_kind,
8375    ///     })),
8376    ///     other => other,
8377    /// }
8378    /// # }
8379    /// ```
8380    pub fn activity_with_options<T: Serialize>(
8381        &self,
8382        activity_type: impl Into<String>,
8383        options: ActivityOptions,
8384        args: T,
8385    ) -> ActivityCall {
8386        ActivityCall {
8387            ctx: self.clone(),
8388            activity_type: activity_type.into(),
8389            options,
8390            args: Some(AvroValue::from_serialize(&args)),
8391            scheduled: false,
8392            parallel_group_path: Vec::new(),
8393        }
8394    }
8395
8396    pub async fn activity_avro_value<T: Serialize>(
8397        &self,
8398        activity_type: impl Into<String>,
8399        args: T,
8400    ) -> Result<AvroValue> {
8401        let mut call = self.activity(activity_type, args);
8402        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8403    }
8404
8405    pub async fn activity_avro_value_with_options<T: Serialize>(
8406        &self,
8407        activity_type: impl Into<String>,
8408        options: ActivityOptions,
8409        args: T,
8410    ) -> Result<AvroValue> {
8411        let mut call = self.activity_with_options(activity_type, options, args);
8412        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8413    }
8414
8415    /// Schedule an activity with a Serde request and decode its Serde result.
8416    pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8417    where
8418        I: Serialize,
8419        O: DeserializeOwned,
8420    {
8421        self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8422            .await
8423    }
8424
8425    /// Schedule an activity with options and decode its result into `O`.
8426    ///
8427    /// Both directions use the fixed Avro Value codec. In particular, this
8428    /// method does not deserialize the JSON-safe inspection projection returned
8429    /// by the dynamic [`ActivityCall`] future.
8430    pub async fn activity_typed_with_options<I, O>(
8431        &self,
8432        activity_type: impl Into<String>,
8433        options: ActivityOptions,
8434        args: I,
8435    ) -> Result<O>
8436    where
8437        I: Serialize,
8438        O: DeserializeOwned,
8439    {
8440        let activity_type = activity_type.into();
8441        let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8442            handler_type_error::<I>(
8443                HandlerKind::Activity,
8444                &activity_type,
8445                HandlerValueKind::Input,
8446                error.to_string(),
8447            )
8448        });
8449        let mut call = ActivityCall {
8450            ctx: self.clone(),
8451            activity_type: activity_type.clone(),
8452            options,
8453            args: Some(encoded),
8454            scheduled: false,
8455            parallel_group_path: Vec::new(),
8456        };
8457        let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8458        decode_handler_result(result, HandlerKind::Activity, &activity_type)
8459    }
8460
8461    /// Schedule and join a deterministic activity/child/timer group.
8462    ///
8463    /// Nested groups retain their input shape. Every durable leaf is scheduled
8464    /// before this future yields, results are assembled by declaration order,
8465    /// and a failure returns [`Error::ParallelFailed`] with typed cause,
8466    /// declaration path, stable group metadata, and completed siblings.
8467    pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8468        ParallelCall::new(self.clone(), operations)
8469    }
8470
8471    /// Alias for [`WorkflowContext::parallel`].
8472    pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8473        self.parallel(operations)
8474    }
8475
8476    /// Lossless fixed-Avro variant of [`WorkflowContext::parallel`].
8477    pub async fn parallel_avro_value(
8478        &self,
8479        operations: Vec<ParallelOperation>,
8480    ) -> Result<Vec<ParallelAvroResult>> {
8481        let mut call = self.parallel(operations);
8482        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8483    }
8484
8485    /// Start every durable operation and resume from the one winner persisted
8486    /// by Server. Non-winning operations continue and remain addressable.
8487    pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8488        let operations = operations
8489            .into_iter()
8490            .enumerate()
8491            .map(|(index, operation)| (SelectionKey::Index(index), operation))
8492            .collect();
8493        SelectCall::new(self.clone(), operations)
8494    }
8495
8496    /// Named-key variant of [`WorkflowContext::select`].
8497    pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8498    where
8499        K: Into<SelectionKey>,
8500    {
8501        SelectCall::new(
8502            self.clone(),
8503            operations
8504                .into_iter()
8505                .map(|(key, operation)| (key.into(), operation))
8506                .collect(),
8507        )
8508    }
8509
8510    /// Create a workflow-local deterministic compensation registry.
8511    pub fn saga(&self) -> Saga {
8512        Saga::new(self.clone())
8513    }
8514
8515    /// Whether this task carries a cooperative cancellation request.
8516    ///
8517    /// Server's current `/cancel` route is terminal and does not set this flag.
8518    /// Service-mode cooperative cancellation is not yet available.
8519    pub fn is_cancellation_requested(&self) -> Result<bool> {
8520        let state = self
8521            .state
8522            .lock()
8523            .map_err(|_| Error::WorkflowStatePoisoned)?;
8524        Ok(state.cancel_requested)
8525    }
8526
8527    /// Raise a typed cooperative cancellation at an author-controlled point.
8528    ///
8529    /// Passing this result to [`Saga::finish`] compensates already registered
8530    /// forward steps before the cancellation remains the initiating outcome.
8531    pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8532        if self.is_cancellation_requested()? {
8533            return Err(Error::WorkflowCancellationRequested(
8534                WorkflowCancellationRequested,
8535            ));
8536        }
8537        Ok(())
8538    }
8539
8540    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8541        SignalCall {
8542            ctx: self.clone(),
8543            signal_name: signal_name.into(),
8544            runtime_reserved_allowed: false,
8545            opened_wait: false,
8546            matched_pending: false,
8547            parallel_group_path: Vec::new(),
8548        }
8549    }
8550
8551    fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8552        SignalCall {
8553            ctx: self.clone(),
8554            signal_name: signal_name.into(),
8555            runtime_reserved_allowed: true,
8556            opened_wait: false,
8557            matched_pending: false,
8558            parallel_group_path: Vec::new(),
8559        }
8560    }
8561
8562    pub async fn wait_signal_avro_value(
8563        &self,
8564        signal_name: impl Into<String>,
8565    ) -> Result<Vec<AvroValue>> {
8566        let mut call = self.wait_signal(signal_name);
8567        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8568    }
8569
8570    /// Return every committed signal argument list with the given name.
8571    ///
8572    /// This history-backed view is deterministic and is intended for
8573    /// condition predicates that must be re-evaluated after a signal while the
8574    /// workflow is blocked on [`WorkflowContext::wait_condition`].
8575    pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8576        self.signals_avro_value(signal_name)?
8577            .into_iter()
8578            .map(|arguments| {
8579                arguments
8580                    .into_iter()
8581                    .map(AvroValue::into_json)
8582                    .collect::<Result<Vec<_>>>()
8583            })
8584            .collect()
8585    }
8586
8587    /// Lossless fixed Avro Value view of committed signals with the given name.
8588    pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8589        let state = self
8590            .state
8591            .lock()
8592            .map_err(|_| Error::WorkflowStatePoisoned)?;
8593        state
8594            .history_events
8595            .iter()
8596            .filter(|event| {
8597                event.event_type == "SignalReceived"
8598                    && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8599            })
8600            .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8601            .collect()
8602    }
8603
8604    /// Return every committed update argument list with the given name.
8605    ///
8606    /// Accepted and applied records for the same update ID are de-duplicated.
8607    /// A Server task created after an update therefore replays the workflow and
8608    /// re-evaluates an open condition without application polling.
8609    pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8610        self.updates_avro_value(update_name)?
8611            .into_iter()
8612            .map(|arguments| {
8613                arguments
8614                    .into_iter()
8615                    .map(AvroValue::into_json)
8616                    .collect::<Result<Vec<_>>>()
8617            })
8618            .collect()
8619    }
8620
8621    /// Lossless fixed Avro Value view of committed updates with the given name.
8622    pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8623        let state = self
8624            .state
8625            .lock()
8626            .map_err(|_| Error::WorkflowStatePoisoned)?;
8627        let mut seen = Vec::new();
8628        let mut updates = Vec::new();
8629        for event in state.history_events.iter() {
8630            if !matches!(
8631                event.event_type.as_str(),
8632                "UpdateAccepted" | "UpdateApplied"
8633            ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8634                || event.payload.get("arguments").is_none()
8635            {
8636                continue;
8637            }
8638            if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8639                if seen.iter().any(|recorded| recorded == update_id) {
8640                    continue;
8641                }
8642                seen.push(update_id.to_string());
8643            }
8644            updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8645        }
8646        Ok(updates)
8647    }
8648
8649    /// Wait for a deterministic predicate to become true or for its durable
8650    /// timeout to elapse.
8651    ///
8652    /// Prefer [`wait_condition!`] for inline predicates so changes to the Rust
8653    /// predicate tokens automatically change the recorded definition
8654    /// fingerprint. Direct callers must provide an equally stable identity in
8655    /// [`ConditionWaitOptions`].
8656    pub fn wait_condition<F>(
8657        &self,
8658        options: ConditionWaitOptions,
8659        predicate: F,
8660    ) -> ConditionWaitCall
8661    where
8662        F: Fn() -> Result<bool> + Send + 'static,
8663    {
8664        ConditionWaitCall {
8665            ctx: self.clone(),
8666            options,
8667            predicate: Box::new(predicate),
8668            occurrence_id: None,
8669            opened_wait: false,
8670            parallel_group_path: Vec::new(),
8671        }
8672    }
8673
8674    /// Wait for server-backed durable time without blocking the worker executor.
8675    ///
8676    /// Polling this future emits one `start_timer` command and yields. The
8677    /// server records the deadline, so neither worker nor server restarts reset
8678    /// the wait. Replay resolves the future only from a `TimerScheduled` and
8679    /// `TimerFired` pair at the same position in the shared durable-command
8680    /// stream, with matching sequence, timer identity, and delay. Sub-second
8681    /// durations round up because protocol deadlines use whole seconds.
8682    ///
8683    /// ```no_run
8684    /// # use durable_workflow::{json, Client, Worker};
8685    /// # use std::time::Duration;
8686    /// # fn configure(client: Client) {
8687    /// let mut worker = Worker::new(client, "rust-workers");
8688    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
8689    ///     ctx.sleep(Duration::from_secs(5)).await?;
8690    ///     Ok(json!({"status": "timer fired"}))
8691    /// });
8692    /// # }
8693    /// ```
8694    pub fn sleep(&self, duration: Duration) -> TimerCall {
8695        let delay_seconds = duration
8696            .as_secs()
8697            .checked_add(u64::from(duration.subsec_nanos() > 0));
8698        TimerCall {
8699            ctx: self.clone(),
8700            delay_seconds,
8701            scheduled: false,
8702            matched_pending: false,
8703            parallel_group_path: Vec::new(),
8704        }
8705    }
8706
8707    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
8708    pub fn start_timer(&self, duration: Duration) -> TimerCall {
8709        self.sleep(duration)
8710    }
8711
8712    /// Evaluate a non-deterministic callback once and durably record its typed value.
8713    ///
8714    /// On replay the callback is not invoked: the value is decoded from the
8715    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
8716    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
8717    /// small values that must remain fixed for the lifetime of a workflow run.
8718    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8719    where
8720        T: Serialize + DeserializeOwned,
8721        F: FnOnce() -> T,
8722    {
8723        {
8724            let mut state = self
8725                .state
8726                .lock()
8727                .map_err(|_| Error::WorkflowStatePoisoned)?;
8728            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8729                return match recorded {
8730                    RecordedCommand::SideEffect { sequence, value } => {
8731                        state.command_cursor += 1;
8732                        value.deserialize().map_err(|error| {
8733                            Error::NonDeterministicReplay(ReplayFailure::new(
8734                                "side_effect_type_mismatch",
8735                                Some(sequence),
8736                                Some(std::any::type_name::<T>().to_string()),
8737                                Some(error.to_string()),
8738                                "recorded side-effect value is incompatible with the requested Rust type",
8739                            ))
8740                        })
8741                    }
8742                    other => Err(command_mismatch(&other, "side effect")),
8743                };
8744            }
8745        }
8746
8747        let value = callback();
8748        let avro_value = AvroValue::from_serialize(&value)?;
8749        let mut state = self
8750            .state
8751            .lock()
8752            .map_err(|_| Error::WorkflowStatePoisoned)?;
8753        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8754        state.commands.push(json!({
8755            "type": "record_side_effect",
8756            "result": result,
8757        }));
8758        Ok(value)
8759    }
8760
8761    /// Record or replay a lossless fixed Avro Value side effect.
8762    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8763    where
8764        F: FnOnce() -> AvroValue,
8765    {
8766        {
8767            let mut state = self
8768                .state
8769                .lock()
8770                .map_err(|_| Error::WorkflowStatePoisoned)?;
8771            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8772                return match recorded {
8773                    RecordedCommand::SideEffect { value, .. } => {
8774                        state.command_cursor += 1;
8775                        Ok(value)
8776                    }
8777                    other => Err(command_mismatch(&other, "side effect")),
8778                };
8779            }
8780        }
8781
8782        let value = callback();
8783        let mut state = self
8784            .state
8785            .lock()
8786            .map_err(|_| Error::WorkflowStatePoisoned)?;
8787        let result = encode_typed_envelope(&value, &state.payload_codec)?;
8788        state.commands.push(json!({
8789            "type": "record_side_effect",
8790            "result": result,
8791        }));
8792        Ok(value)
8793    }
8794
8795    /// Append output items at a deterministic workflow command boundary.
8796    ///
8797    /// Stable idempotency keys are derived from the server-provided durable
8798    /// workflow command identity, command ordinal, and item index. Replay
8799    /// consumes the recorded side effect and never emits another append.
8800    pub fn append_workflow_stream(
8801        &self,
8802        stream_name: impl Into<String>,
8803        items: &[WorkflowStreamAppendItem],
8804        max_pending_items: Option<u64>,
8805    ) -> Result<()> {
8806        if items.is_empty() {
8807            return Err(Error::Codec(
8808                "workflow_stream_items_empty: append requires at least one item".to_string(),
8809            ));
8810        }
8811        if max_pending_items == Some(0) {
8812            return Err(Error::Codec(
8813                "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8814                    .to_string(),
8815            ));
8816        }
8817        let stream_name = stream_name.into();
8818        if stream_name.is_empty() {
8819            return Err(Error::Codec(
8820                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8821            ));
8822        }
8823
8824        let mut state = self
8825            .state
8826            .lock()
8827            .map_err(|_| Error::WorkflowStatePoisoned)?;
8828        let command_ordinal = state.workflow_stream_command_counter;
8829        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8830            state.workflow_stream_command_counter += 1;
8831            return match recorded {
8832                RecordedCommand::SideEffect { .. } => {
8833                    state.command_cursor += 1;
8834                    Ok(())
8835                }
8836                other => Err(command_mismatch(&other, "workflow stream append")),
8837            };
8838        }
8839
8840        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8841        state.workflow_stream_command_counter += 1;
8842        let wire_items = items
8843            .iter()
8844            .enumerate()
8845            .map(|(item_index, item)| {
8846                item.wire_value(Some(format!(
8847                    "dw-stream:{identity}:{command_ordinal}:{item_index}"
8848                )))
8849            })
8850            .collect::<Vec<_>>();
8851        let mut directive = json!({
8852            "operation": "append",
8853            "stream_name": stream_name,
8854            "command_identity": identity,
8855            "command_ordinal": command_ordinal,
8856            "items": wire_items,
8857        });
8858        if let Some(max_pending_items) = max_pending_items {
8859            directive["max_pending_items"] = json!(max_pending_items);
8860        }
8861        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8862        state.commands.push(json!({
8863            "type": "record_side_effect",
8864            "result": result,
8865            "workflow_stream": directive,
8866        }));
8867        Ok(())
8868    }
8869
8870    /// Close a run-scoped output stream at a deterministic command boundary.
8871    pub fn close_workflow_stream(
8872        &self,
8873        stream_name: impl Into<String>,
8874        retention_seconds: Option<u64>,
8875    ) -> Result<()> {
8876        self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8877    }
8878
8879    /// Mark a run-scoped output stream errored at a deterministic command boundary.
8880    pub fn error_workflow_stream(
8881        &self,
8882        stream_name: impl Into<String>,
8883        error_reason: impl Into<String>,
8884        retention_seconds: Option<u64>,
8885    ) -> Result<()> {
8886        let error_reason = error_reason.into();
8887        if error_reason.is_empty() {
8888            return Err(Error::Codec(
8889                "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8890            ));
8891        }
8892        self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8893    }
8894
8895    fn finish_workflow_stream(
8896        &self,
8897        stream_name: String,
8898        error_reason: Option<String>,
8899        retention_seconds: Option<u64>,
8900    ) -> Result<()> {
8901        if stream_name.is_empty() {
8902            return Err(Error::Codec(
8903                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8904            ));
8905        }
8906        if retention_seconds == Some(0) {
8907            return Err(Error::Codec(
8908                "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8909            ));
8910        }
8911        let mut state = self
8912            .state
8913            .lock()
8914            .map_err(|_| Error::WorkflowStatePoisoned)?;
8915        let command_ordinal = state.workflow_stream_command_counter;
8916        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8917            state.workflow_stream_command_counter += 1;
8918            return match recorded {
8919                RecordedCommand::SideEffect { .. } => {
8920                    state.command_cursor += 1;
8921                    Ok(())
8922                }
8923                other => Err(command_mismatch(&other, "workflow stream close")),
8924            };
8925        }
8926        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8927        state.workflow_stream_command_counter += 1;
8928        let mut directive = json!({
8929            "operation": if error_reason.is_some() { "error" } else { "close" },
8930            "stream_name": stream_name,
8931            "command_identity": identity,
8932            "command_ordinal": command_ordinal,
8933        });
8934        if let Some(error_reason) = error_reason {
8935            directive["error_reason"] = json!(error_reason);
8936        }
8937        if let Some(retention_seconds) = retention_seconds {
8938            directive["retention_seconds"] = json!(retention_seconds);
8939        }
8940        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8941        state.commands.push(json!({
8942            "type": "record_side_effect",
8943            "result": result,
8944            "workflow_stream": directive,
8945        }));
8946        Ok(())
8947    }
8948
8949    fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8950        let identity = state.workflow_command_identity.as_str();
8951        if identity.is_empty() {
8952            return Err(Error::MissingWorkflowCommandIdentity);
8953        }
8954        Ok(identity)
8955    }
8956
8957    /// Validate, emit, or replay a typed workflow search-attribute update.
8958    ///
8959    /// The command is non-blocking within a workflow decision, but its
8960    /// `SearchAttributesUpserted` event occupies the same deterministic command
8961    /// stream as activities, timers, conditions, and other durable operations.
8962    pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8963        update.validate()?;
8964        let (attributes, attribute_types) = update.into_wire_parts();
8965        let mut state = self
8966            .state
8967            .lock()
8968            .map_err(|_| Error::WorkflowStatePoisoned)?;
8969
8970        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8971            return match recorded {
8972                RecordedCommand::SearchAttributes {
8973                    sequence,
8974                    attributes: recorded_attributes,
8975                    attribute_types: recorded_attribute_types,
8976                } => {
8977                    if recorded_attributes != attributes {
8978                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8979                            "search_attribute_value_mismatch",
8980                            Some(sequence),
8981                            Some(recorded_attributes.to_string()),
8982                            Some(attributes.to_string()),
8983                            "search-attribute values differ from the recorded durable command",
8984                        )));
8985                    }
8986                    if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8987                        if recorded_types != attribute_types {
8988                            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8989                                "search_attribute_type_mismatch",
8990                                Some(sequence),
8991                                Some(json!(recorded_types).to_string()),
8992                                Some(json!(attribute_types).to_string()),
8993                                "search-attribute declared types differ from the recorded durable command",
8994                            )));
8995                        }
8996                    }
8997                    state.command_cursor += 1;
8998                    Ok(())
8999                }
9000                other => Err(command_mismatch(&other, "search-attribute update")),
9001            };
9002        }
9003
9004        let mut command = serde_json::Map::from_iter([
9005            ("type".to_string(), json!("upsert_search_attributes")),
9006            ("attributes".to_string(), attributes),
9007        ]);
9008        if !attribute_types.is_empty() {
9009            command.insert("attribute_types".to_string(), json!(attribute_types));
9010        }
9011        state.commands.push(Value::Object(command));
9012        Ok(())
9013    }
9014
9015    /// Record a UUIDv4 once and return the same UUID on every replay.
9016    pub fn uuid_v4(&self) -> Result<Uuid> {
9017        self.side_effect(Uuid::new_v4)
9018    }
9019
9020    /// Select the newest supported version for a change, or replay the version
9021    /// already committed for that stable change ID.
9022    pub fn get_version(
9023        &self,
9024        change_id: impl Into<String>,
9025        min_supported: i32,
9026        max_supported: i32,
9027    ) -> Result<i32> {
9028        let change_id = change_id.into();
9029        if change_id.trim().is_empty() {
9030            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9031                "version_change_id_invalid",
9032                None,
9033                Some("non-empty change ID".to_string()),
9034                Some(change_id),
9035                "version markers require a stable non-empty change ID",
9036            )));
9037        }
9038        if min_supported > max_supported {
9039            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9040                "version_range_invalid",
9041                None,
9042                Some("min_supported <= max_supported".to_string()),
9043                Some(format!("{min_supported}..={max_supported}")),
9044                "version marker supported range is invalid",
9045            )));
9046        }
9047
9048        let mut state = self
9049            .state
9050            .lock()
9051            .map_err(|_| Error::WorkflowStatePoisoned)?;
9052        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
9053            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
9054            return Ok(version);
9055        }
9056
9057        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9058            return match recorded {
9059                RecordedCommand::VersionMarker {
9060                    sequence,
9061                    change_id: recorded_change_id,
9062                    version,
9063                    ..
9064                } => {
9065                    if recorded_change_id != change_id {
9066                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9067                            "version_change_id_mismatch",
9068                            Some(sequence),
9069                            Some(recorded_change_id),
9070                            Some(change_id),
9071                            "recorded version marker change ID differs from current workflow code",
9072                        )));
9073                    }
9074                    ensure_version_supported(
9075                        &change_id,
9076                        version,
9077                        min_supported,
9078                        max_supported,
9079                        sequence,
9080                    )?;
9081                    state.command_cursor += 1;
9082                    state.version_markers.insert(change_id, (version, sequence));
9083                    Ok(version)
9084                }
9085                other => Err(command_mismatch(
9086                    &other,
9087                    format!("version marker:{change_id}"),
9088                )),
9089            };
9090        }
9091
9092        let version = max_supported;
9093        state.commands.push(json!({
9094            "type": "record_version_marker",
9095            "change_id": change_id,
9096            "version": version,
9097            "min_supported": min_supported,
9098            "max_supported": max_supported,
9099        }));
9100        // Sequence numbers are assigned by the server. Zero identifies a marker
9101        // selected in this uncommitted decision batch for duplicate-call checks.
9102        state.version_markers.insert(change_id, (version, 0));
9103        Ok(version)
9104    }
9105
9106    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
9107    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
9108        Ok(self.get_version(change_id, -1, 1)? == 1)
9109    }
9110
9111    /// Keep a patch marker in history after the legacy branch has been removed.
9112    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
9113        self.get_version(change_id, -1, 1).map(|_| ())
9114    }
9115
9116    /// Merge non-indexed workflow memo metadata through durable history.
9117    ///
9118    /// Avro `null` deletes a key. The SDK encodes the complete patch in the
9119    /// public Avro payload envelope consumed by Server and Cloud runtimes.
9120    pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
9121        let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
9122        let mut state = self
9123            .state
9124            .lock()
9125            .map_err(|_| Error::WorkflowStatePoisoned)?;
9126
9127        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9128            return match recorded {
9129                RecordedCommand::Memo {
9130                    sequence,
9131                    entries: recorded_entries,
9132                } => {
9133                    if recorded_entries != entries {
9134                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9135                            "memo_update_mismatch",
9136                            Some(sequence),
9137                            Some(format!("{recorded_entries:?}")),
9138                            Some(format!("{entries:?}")),
9139                            "recorded memo entries differ from the current workflow update",
9140                        )));
9141                    }
9142                    state.command_cursor += 1;
9143                    Ok(())
9144                }
9145                other => Err(command_mismatch(&other, "memo upsert")),
9146            };
9147        }
9148
9149        let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
9150        state.commands.push(json!({
9151            "type": "upsert_memo",
9152            "entries": entries_envelope,
9153        }));
9154        Ok(())
9155    }
9156
9157    /// Start a named durable child on an explicit queue and await its result.
9158    ///
9159    /// The command is recorded in the parent's sequence-ordered durable command
9160    /// stream. Replay keeps a scheduled child pending without emitting another
9161    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
9162    /// values preserve the history payload codec and include both sides of the
9163    /// durable relationship; failures are returned as
9164    /// [`Error::ChildWorkflowFailed`].
9165    ///
9166    /// ```no_run
9167    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
9168    /// # fn configure(client: Client) {
9169    /// let mut worker = Worker::new(client, "parent-workers");
9170    /// worker.register_workflow("order-parent", |ctx, _input| async move {
9171    ///     let child = ctx
9172    ///         .start_child_workflow(
9173    ///             "fulfil-order",
9174    ///             ChildWorkflowOptions::new("fulfilment-workers")
9175    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
9176    ///             json!([{"order_id": "order-42"}]),
9177    ///         )
9178    ///         .await?;
9179    ///     Ok(child.result)
9180    /// });
9181    /// # }
9182    /// ```
9183    pub fn start_child_workflow<T: Serialize>(
9184        &self,
9185        workflow_type: impl Into<String>,
9186        options: ChildWorkflowOptions,
9187        args: T,
9188    ) -> ChildWorkflowCall {
9189        ChildWorkflowCall {
9190            ctx: self.clone(),
9191            workflow_type: workflow_type.into(),
9192            options,
9193            args: Some(AvroValue::from_serialize(&args)),
9194            scheduled: false,
9195            matched_pending: false,
9196            parallel_group_path: Vec::new(),
9197        }
9198    }
9199
9200    pub async fn start_child_workflow_avro_value<T: Serialize>(
9201        &self,
9202        workflow_type: impl Into<String>,
9203        options: ChildWorkflowOptions,
9204        args: T,
9205    ) -> Result<ChildWorkflowAvroResult> {
9206        let mut call = self.start_child_workflow(workflow_type, options, args);
9207        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9208    }
9209
9210    fn take_commands(&self) -> Result<Vec<Value>> {
9211        let mut state = self
9212            .state
9213            .lock()
9214            .map_err(|_| Error::WorkflowStatePoisoned)?;
9215        Ok(std::mem::take(&mut state.commands))
9216    }
9217
9218    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
9219        let mut state = self
9220            .state
9221            .lock()
9222            .map_err(|_| Error::WorkflowStatePoisoned)?;
9223
9224        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9225            return Err(command_mismatch(&recorded, "continue as new"));
9226        }
9227        if state.recorded_continue_as_new_sequence.is_some() {
9228            state.continue_as_new_consumed = true;
9229            return Ok(None);
9230        }
9231
9232        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
9233        let mut command = serde_json::Map::from_iter([
9234            ("type".to_string(), json!("continue_as_new")),
9235            ("arguments".to_string(), arguments),
9236            ("queue".to_string(), json!(state.task_queue.clone())),
9237        ]);
9238        if let Some(workflow_type) = request.options.workflow_type {
9239            command.insert("workflow_type".to_string(), json!(workflow_type));
9240        }
9241        if let Some(task_queue) = request.options.task_queue {
9242            command.insert("queue".to_string(), json!(task_queue));
9243        }
9244        Ok(Some(Value::Object(command)))
9245    }
9246
9247    fn matched_recorded_pending(&self) -> Result<bool> {
9248        let state = self
9249            .state
9250            .lock()
9251            .map_err(|_| Error::WorkflowStatePoisoned)?;
9252        Ok(state.matched_recorded_pending)
9253    }
9254
9255    fn ensure_history_consumed(&self) -> Result<()> {
9256        let state = self
9257            .state
9258            .lock()
9259            .map_err(|_| Error::WorkflowStatePoisoned)?;
9260        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
9261            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9262                "recorded_commands_unconsumed",
9263                Some(command.sequence()),
9264                Some(command.shape().to_string()),
9265                Some("workflow completion".to_string()),
9266                "workflow completed before consuming all recorded durable commands",
9267            )));
9268        }
9269        if let Some(sequence) = state
9270            .recorded_continue_as_new_sequence
9271            .filter(|_| !state.continue_as_new_consumed)
9272        {
9273            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9274                "recorded_continue_as_new_unconsumed",
9275                Some(sequence),
9276                Some("continue as new".to_string()),
9277                Some("workflow completion".to_string()),
9278                "workflow completed without consuming its recorded continue-as-new transition",
9279            )));
9280        }
9281        Ok(())
9282    }
9283}
9284
9285fn contiguous_message_stream_count(
9286    pending: &[MessageStreamMessage],
9287    cursor: u64,
9288    max_items: usize,
9289) -> usize {
9290    pending
9291        .iter()
9292        .take(max_items)
9293        .enumerate()
9294        .take_while(|(offset, message)| {
9295            u64::try_from(*offset)
9296                .ok()
9297                .and_then(|offset| cursor.checked_add(offset + 1))
9298                == Some(message.position)
9299        })
9300        .count()
9301}
9302
9303fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
9304    matches!(
9305        event.event_type.as_str(),
9306        "ActivityScheduled"
9307            | "TimerScheduled"
9308            | "ChildWorkflowScheduled"
9309            | "SignalWaitOpened"
9310            | "ConditionWaitOpened"
9311            | "SearchAttributesUpserted"
9312            | "SideEffectRecorded"
9313            | "VersionMarkerRecorded"
9314            | "MemoUpserted"
9315            | "WorkflowContinuedAsNew"
9316    )
9317}
9318
9319#[derive(Debug)]
9320struct WorkflowState {
9321    workflow_id: Option<String>,
9322    run_id: Option<String>,
9323    task_queue: String,
9324    payload_codec: String,
9325    history_events: Arc<Vec<HistoryEvent>>,
9326    history_budget: WorkflowHistoryBudget,
9327    cancel_requested: bool,
9328    resume_signal: Option<ResumeSignal>,
9329    recorded_commands: Vec<RecordedCommand>,
9330    selection_markers: Vec<SelectionMarker>,
9331    selection_marker_cursor: usize,
9332    cancelled_selection_members: Vec<SelectionCancellation>,
9333    recorded_continue_as_new_sequence: Option<u64>,
9334    continue_as_new_consumed: bool,
9335    command_cursor: usize,
9336    condition_wait_occurrence_counter: u64,
9337    matched_recorded_pending: bool,
9338    version_markers: HashMap<String, (i32, u64)>,
9339    workflow_command_identity: String,
9340    workflow_stream_command_counter: u64,
9341    commands: Vec<Value>,
9342    message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9343    message_stream_cursors: HashMap<String, u64>,
9344    message_stream_waits: HashMap<String, u64>,
9345}
9346
9347impl WorkflowState {
9348    #[cfg(test)]
9349    fn new(
9350        history: Vec<HistoryEvent>,
9351        task_queue: String,
9352        payload_codec: String,
9353        resume_signal: Option<ResumeSignal>,
9354    ) -> Result<Self> {
9355        Self::new_with_identity(
9356            history,
9357            None,
9358            None,
9359            task_queue,
9360            payload_codec,
9361            resume_signal,
9362        )
9363    }
9364
9365    fn new_with_identity(
9366        history: Vec<HistoryEvent>,
9367        workflow_id: Option<String>,
9368        run_id: Option<String>,
9369        task_queue: String,
9370        payload_codec: String,
9371        resume_signal: Option<ResumeSignal>,
9372    ) -> Result<Self> {
9373        let recorded_commands = recorded_commands(
9374            &history,
9375            &payload_codec,
9376            WorkflowIdentity {
9377                workflow_id: workflow_id.clone(),
9378                run_id: run_id.clone(),
9379            },
9380        )?;
9381        let selection_markers = recorded_selection_markers(&history)?;
9382        let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9383        let recorded_continue_as_new = history
9384            .iter()
9385            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9386            .collect::<Vec<_>>();
9387        if recorded_continue_as_new.len() > 1 {
9388            return Err(invalid_recorded_history(
9389                "duplicate_continue_as_new_transition",
9390                recorded_continue_as_new
9391                    .last()
9392                    .and_then(|event| durable_event_sequence(event))
9393                    .unwrap_or(0),
9394                "one WorkflowContinuedAsNew event",
9395                &format!(
9396                    "{} WorkflowContinuedAsNew events",
9397                    recorded_continue_as_new.len()
9398                ),
9399                "workflow history records one continue-as-new transition more than once",
9400            ));
9401        }
9402        let recorded_continue_as_new_sequence = recorded_continue_as_new
9403            .first()
9404            .map(|event| {
9405                durable_event_sequence(event).ok_or_else(|| {
9406                    Error::NonDeterministicReplay(ReplayFailure::new(
9407                        "continue_as_new_sequence_missing",
9408                        None,
9409                        Some("recorded transition sequence".to_string()),
9410                        Some("missing sequence".to_string()),
9411                        "WorkflowContinuedAsNew history is missing its recorded sequence",
9412                    ))
9413                })
9414            })
9415            .transpose()?;
9416        let mut message_stream_cursors = HashMap::new();
9417        for event in &history {
9418            if !matches!(
9419                event.event_type.as_str(),
9420                "SignalReceived" | "SignalApplied"
9421            ) || event.payload.get("signal_name").and_then(Value::as_str)
9422                != Some(MESSAGE_STREAM_SIGNAL)
9423            {
9424                continue;
9425            }
9426            let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9427            if arguments.len() != 1 {
9428                continue;
9429            }
9430            let envelope = arguments[0].clone().into_json()?;
9431            let Some(envelope) = envelope.as_object() else {
9432                continue;
9433            };
9434            if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9435            {
9436                continue;
9437            }
9438            let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9439                continue;
9440            };
9441            let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9442            else {
9443                continue;
9444            };
9445            let cursor = message_stream_cursors
9446                .entry(stream_name.to_string())
9447                .or_insert(0);
9448            *cursor = (*cursor).max(through_position);
9449        }
9450        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9451        let cancel_requested = history.iter().any(|event| {
9452            matches!(
9453                event.event_type.as_str(),
9454                "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9455            )
9456        });
9457        Ok(Self {
9458            workflow_command_identity: String::new(),
9459            workflow_stream_command_counter: 0,
9460            workflow_id,
9461            run_id,
9462            task_queue,
9463            payload_codec,
9464            history_events: Arc::new(history),
9465            history_budget: WorkflowHistoryBudget {
9466                event_count,
9467                ..WorkflowHistoryBudget::default()
9468            },
9469            cancel_requested,
9470            resume_signal,
9471            recorded_commands,
9472            selection_markers,
9473            selection_marker_cursor: 0,
9474            cancelled_selection_members,
9475            recorded_continue_as_new_sequence,
9476            continue_as_new_consumed: false,
9477            command_cursor: 0,
9478            condition_wait_occurrence_counter: 0,
9479            matched_recorded_pending: false,
9480            version_markers: HashMap::new(),
9481            commands: Vec::new(),
9482            message_stream_messages: HashMap::new(),
9483            message_stream_cursors,
9484            message_stream_waits: HashMap::new(),
9485        })
9486    }
9487}
9488
9489enum MessageStreamDelivery {
9490    Message(MessageStreamMessage),
9491    Cursor {
9492        stream_name: String,
9493        through_position: u64,
9494    },
9495}
9496
9497fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9498    if arguments.len() != 1 {
9499        return Ok(None);
9500    }
9501    let envelope = arguments
9502        .into_iter()
9503        .next()
9504        .expect("one argument was checked");
9505    let Some(envelope) = envelope.as_object() else {
9506        return Ok(None);
9507    };
9508    let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9509        return Ok(None);
9510    };
9511    if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9512        let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9513        else {
9514            return Ok(None);
9515        };
9516        return Ok(Some(MessageStreamDelivery::Cursor {
9517            stream_name: stream_name.to_string(),
9518            through_position,
9519        }));
9520    }
9521    if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9522        return Ok(None);
9523    }
9524    let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9525        return Ok(None);
9526    };
9527    let Some(position) = envelope
9528        .get("position")
9529        .and_then(Value::as_u64)
9530        .filter(|value| *value > 0)
9531    else {
9532        return Ok(None);
9533    };
9534    let Some(payload_envelope) = envelope.get("payload_envelope") else {
9535        return Ok(None);
9536    };
9537    let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9538    else {
9539        return Ok(None);
9540    };
9541    let decoded = decode_avro_value(&payload_envelope)?;
9542    let AvroValue::Array(values) = decoded else {
9543        return Ok(None);
9544    };
9545    Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9546        stream_name: stream_name.to_string(),
9547        message_id: message_id.to_string(),
9548        position,
9549        arguments: values,
9550    })))
9551}
9552
9553#[derive(Clone, Debug)]
9554enum RecordedCommand {
9555    Activity {
9556        sequence: u64,
9557        activity_type: Option<String>,
9558        options: Option<RecordedActivityOptions>,
9559        outcome: Option<ActivityOutcome>,
9560        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9561    },
9562    Timer {
9563        sequence: u64,
9564        delay_seconds: u64,
9565        fired: bool,
9566        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9567    },
9568    ChildWorkflow {
9569        sequence: u64,
9570        workflow_type: Option<String>,
9571        outcome: Option<ChildWorkflowOutcome>,
9572        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9573    },
9574    SignalWait {
9575        sequence: u64,
9576        signal_name: String,
9577        value: Option<Vec<AvroValue>>,
9578        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9579    },
9580    ConditionWait {
9581        sequence: u64,
9582        occurrence_id: String,
9583        condition_key: Option<String>,
9584        predicate_identity: String,
9585        timeout_seconds: Option<u64>,
9586        result: Option<ConditionWaitResult>,
9587        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9588    },
9589    SearchAttributes {
9590        sequence: u64,
9591        attributes: Value,
9592        attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9593    },
9594    SideEffect {
9595        sequence: u64,
9596        value: AvroValue,
9597    },
9598    VersionMarker {
9599        sequence: u64,
9600        change_id: String,
9601        version: i32,
9602    },
9603    Memo {
9604        sequence: u64,
9605        entries: AvroValue,
9606    },
9607}
9608
9609#[derive(Clone, Debug, PartialEq, Eq)]
9610struct SelectionMarker {
9611    selection_group_id: String,
9612    selection_group_base_sequence: u64,
9613    selection_group_size: usize,
9614    member_key: SelectionKey,
9615    member_index: usize,
9616    member_base_sequence: u64,
9617    member_size: usize,
9618    operation_kind: String,
9619    operation_identity: String,
9620    outcome: String,
9621    resolution_event_id: String,
9622    resolution_event_type: String,
9623}
9624
9625#[derive(Clone, Debug, PartialEq, Eq)]
9626struct SelectionCancellation {
9627    selection_group_id: String,
9628    member_key: SelectionKey,
9629    member_index: usize,
9630    member_base_sequence: u64,
9631    member_size: usize,
9632    operation_kind: String,
9633    operation_identity: String,
9634}
9635
9636fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9637    let mut markers: Vec<SelectionMarker> = Vec::new();
9638    for event in events
9639        .iter()
9640        .filter(|event| event.event_type == "SelectionResolved")
9641    {
9642        let payload = &event.payload;
9643        let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9644        let group_size = required_selection_usize(payload, "selection_group_size")?;
9645        let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9646        let member_size = required_selection_usize(payload, "member_size")?;
9647        let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9648        let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9649            invalid_recorded_history(
9650                "selection_marker_invalid",
9651                base_sequence,
9652                "non-empty selection_group_id",
9653                &payload.to_string(),
9654                "selection winner history is missing its durable group identity",
9655            )
9656        })?;
9657        let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9658        if group_id != expected_group_id {
9659            return Err(invalid_recorded_history(
9660                "selection_marker_invalid",
9661                base_sequence,
9662                &expected_group_id,
9663                &group_id,
9664                "selection winner history contains an incompatible group identity",
9665            ));
9666        }
9667        let group_end = base_sequence
9668            .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9669            .unwrap_or(u64::MAX);
9670        let member_end = member_base_sequence
9671            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9672            .unwrap_or(u64::MAX);
9673        if member_index >= group_size
9674            || member_base_sequence < base_sequence
9675            || member_end > group_end
9676        {
9677            return Err(invalid_recorded_history(
9678                "selection_marker_invalid",
9679                base_sequence,
9680                "winner member within selection group bounds",
9681                &payload.to_string(),
9682                "selection winner history contains an invalid member range",
9683            ));
9684        }
9685        let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9686            invalid_recorded_history(
9687                "selection_marker_invalid",
9688                base_sequence,
9689                "selection operation kind",
9690                &payload.to_string(),
9691                "selection winner history is missing its operation kind",
9692            )
9693        })?;
9694        if !matches!(
9695            operation_kind.as_str(),
9696            "activity" | "child" | "timer" | "signal" | "condition" | "group"
9697        ) {
9698            return Err(invalid_recorded_history(
9699                "selection_marker_invalid",
9700                base_sequence,
9701                "activity, child, timer, signal, condition, or group",
9702                &operation_kind,
9703                "selection winner history contains an unsupported operation kind",
9704            ));
9705        }
9706        let operation_identity =
9707            payload_string(payload, "operation_identity").ok_or_else(|| {
9708                invalid_recorded_history(
9709                    "selection_marker_invalid",
9710                    base_sequence,
9711                    "non-empty operation identity",
9712                    &payload.to_string(),
9713                    "selection winner history is missing its durable operation identity",
9714                )
9715            })?;
9716        let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9717            invalid_recorded_history(
9718                "selection_marker_invalid",
9719                base_sequence,
9720                "completed or failed selection outcome",
9721                &payload.to_string(),
9722                "selection winner history is missing its outcome",
9723            )
9724        })?;
9725        if !matches!(outcome.as_str(), "completed" | "failed") {
9726            return Err(invalid_recorded_history(
9727                "selection_marker_invalid",
9728                base_sequence,
9729                "completed or failed selection outcome",
9730                &outcome,
9731                "selection winner history contains an unsupported outcome",
9732            ));
9733        }
9734        let marker = SelectionMarker {
9735            selection_group_id: group_id,
9736            selection_group_base_sequence: base_sequence,
9737            selection_group_size: group_size,
9738            member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9739            member_index,
9740            member_base_sequence,
9741            member_size,
9742            operation_kind,
9743            operation_identity,
9744            outcome,
9745            resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9746                || {
9747                    invalid_recorded_history(
9748                        "selection_marker_invalid",
9749                        base_sequence,
9750                        "durable resolution_event_id",
9751                        &payload.to_string(),
9752                        "selection winner history is missing its terminal event identity",
9753                    )
9754                },
9755            )?,
9756            resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9757                || {
9758                    invalid_recorded_history(
9759                        "selection_marker_invalid",
9760                        base_sequence,
9761                        "durable resolution_event_type",
9762                        &payload.to_string(),
9763                        "selection winner history is missing its terminal event type",
9764                    )
9765                },
9766            )?,
9767        };
9768        if let Some(existing) = markers
9769            .iter()
9770            .find(|existing| existing.selection_group_id == marker.selection_group_id)
9771        {
9772            if existing != &marker {
9773                return Err(invalid_recorded_history(
9774                    "selection_marker_conflict",
9775                    base_sequence,
9776                    &format!("one winner for {}", marker.selection_group_id),
9777                    &payload.to_string(),
9778                    "selection history records conflicting winners for one durable group",
9779                ));
9780            }
9781            continue;
9782        }
9783        markers.push(marker);
9784    }
9785    Ok(markers)
9786}
9787
9788fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9789    let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9790    for event in events
9791        .iter()
9792        .filter(|event| event.event_type == "SelectionOperationCancelled")
9793    {
9794        let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9795            invalid_recorded_history(
9796                "selection_cancellation_invalid",
9797                0,
9798                "non-empty selection_group_id",
9799                &event.payload.to_string(),
9800                "selection cancellation history is missing its group identity",
9801            )
9802        })?;
9803        let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9804        let marker = SelectionCancellation {
9805            selection_group_id: group_id,
9806            member_key: selection_key_from_value(
9807                event.payload.get("member_key"),
9808                member_base_sequence,
9809            )?,
9810            member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9811            member_base_sequence,
9812            member_size: required_selection_usize(&event.payload, "member_size")?,
9813            operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9814                invalid_recorded_history(
9815                    "selection_cancellation_invalid",
9816                    member_base_sequence,
9817                    "selection operation kind",
9818                    &event.payload.to_string(),
9819                    "selection cancellation is missing its operation kind",
9820                )
9821            })?,
9822            operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9823                || {
9824                    invalid_recorded_history(
9825                        "selection_cancellation_invalid",
9826                        member_base_sequence,
9827                        "selection operation identity",
9828                        &event.payload.to_string(),
9829                        "selection cancellation is missing its operation identity",
9830                    )
9831                },
9832            )?,
9833        };
9834        if let Some(existing) = cancelled.iter().find(|recorded| {
9835            recorded.selection_group_id == marker.selection_group_id
9836                && recorded.member_base_sequence == marker.member_base_sequence
9837        }) {
9838            if existing != &marker {
9839                return Err(invalid_recorded_history(
9840                    "selection_cancellation_conflict",
9841                    member_base_sequence,
9842                    "one stable SelectionOperationCancelled marker",
9843                    &event.payload.to_string(),
9844                    "selection cancellation history contains conflicting member metadata",
9845                ));
9846            }
9847        } else {
9848            cancelled.push(marker);
9849        }
9850    }
9851    Ok(cancelled)
9852}
9853
9854fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9855    payload
9856        .get(field)
9857        .and_then(value_as_u64)
9858        .filter(|value| *value > 0)
9859        .ok_or_else(|| {
9860            invalid_recorded_history(
9861                "selection_marker_invalid",
9862                0,
9863                &format!("positive integer {field}"),
9864                &payload.to_string(),
9865                "selection history contains invalid durable identity metadata",
9866            )
9867        })
9868}
9869
9870fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9871    required_selection_usize_allow_zero(payload, field).and_then(|value| {
9872        if value > 0 {
9873            Ok(value)
9874        } else {
9875            Err(invalid_recorded_history(
9876                "selection_marker_invalid",
9877                0,
9878                &format!("positive integer {field}"),
9879                &payload.to_string(),
9880                "selection history contains invalid durable identity metadata",
9881            ))
9882        }
9883    })
9884}
9885
9886fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9887    payload
9888        .get(field)
9889        .and_then(value_as_u64)
9890        .and_then(|value| usize::try_from(value).ok())
9891        .ok_or_else(|| {
9892            invalid_recorded_history(
9893                "selection_marker_invalid",
9894                0,
9895                &format!("non-negative integer {field}"),
9896                &payload.to_string(),
9897                "selection history contains invalid durable identity metadata",
9898            )
9899        })
9900}
9901
9902#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9903struct RecordedActivityOptions {
9904    task_queue: RecordedSnapshotValue<Option<String>>,
9905    execution_mode: RecordedSnapshotValue<Option<String>>,
9906    retry_policy: ActivityRetrySnapshot,
9907}
9908
9909#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9910enum RecordedSnapshotValue<T> {
9911    /// Older history did not persist this field, so it cannot constrain replay.
9912    Unknown,
9913    Known(T),
9914}
9915
9916impl<T: PartialEq> RecordedSnapshotValue<T> {
9917    fn matches_current(&self, current: &Self) -> bool {
9918        match self {
9919            Self::Unknown => true,
9920            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9921        }
9922    }
9923}
9924
9925#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9926struct ActivityRetrySnapshot {
9927    snapshot_version: RecordedSnapshotValue<Option<u64>>,
9928    max_attempts: RecordedSnapshotValue<Option<u64>>,
9929    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9930    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9931    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9932    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9933    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9934    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9935}
9936
9937impl ActivityRetrySnapshot {
9938    fn matches_current(&self, current: &Self) -> bool {
9939        self.snapshot_version
9940            .matches_current(&current.snapshot_version)
9941            && self.max_attempts.matches_current(&current.max_attempts)
9942            && self
9943                .backoff_seconds
9944                .matches_current(&current.backoff_seconds)
9945            && self
9946                .start_to_close_timeout
9947                .matches_current(&current.start_to_close_timeout)
9948            && self
9949                .schedule_to_start_timeout
9950                .matches_current(&current.schedule_to_start_timeout)
9951            && self
9952                .schedule_to_close_timeout
9953                .matches_current(&current.schedule_to_close_timeout)
9954            && self
9955                .heartbeat_timeout
9956                .matches_current(&current.heartbeat_timeout)
9957            && self
9958                .non_retryable_error_types
9959                .matches_current(&current.non_retryable_error_types)
9960    }
9961}
9962
9963fn recorded_optional_u64(
9964    object: Option<&serde_json::Map<String, Value>>,
9965    field: &str,
9966) -> RecordedSnapshotValue<Option<u64>> {
9967    match object.and_then(|object| object.get(field)) {
9968        None => RecordedSnapshotValue::Unknown,
9969        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9970        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9971    }
9972}
9973
9974fn recorded_optional_string(
9975    object: &serde_json::Map<String, Value>,
9976    field: &str,
9977) -> RecordedSnapshotValue<Option<String>> {
9978    match object.get(field) {
9979        None => RecordedSnapshotValue::Unknown,
9980        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9981        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9982    }
9983}
9984
9985fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9986    let policy = policy.and_then(Value::as_object);
9987    let backoff_seconds = policy
9988        .and_then(|policy| policy.get("backoff_seconds"))
9989        .and_then(Value::as_array)
9990        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9991        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9992    let mut non_retryable_error_types = Vec::new();
9993    for error_type in policy
9994        .and_then(|policy| policy.get("non_retryable_error_types"))
9995        .and_then(Value::as_array)
9996        .into_iter()
9997        .flatten()
9998        .filter_map(Value::as_str)
9999        .map(str::trim)
10000        .filter(|error_type| !error_type.is_empty())
10001    {
10002        if !non_retryable_error_types
10003            .iter()
10004            .any(|recorded| recorded == error_type)
10005        {
10006            non_retryable_error_types.push(error_type.to_string());
10007        }
10008    }
10009
10010    ActivityRetrySnapshot {
10011        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
10012        max_attempts: recorded_optional_u64(policy, "max_attempts"),
10013        backoff_seconds,
10014        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
10015        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
10016        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
10017        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
10018        non_retryable_error_types: if policy
10019            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
10020        {
10021            RecordedSnapshotValue::Known(non_retryable_error_types)
10022        } else {
10023            RecordedSnapshotValue::Unknown
10024        },
10025    }
10026}
10027
10028fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
10029    let policy = options.retry_policy.as_ref();
10030    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
10031        Some(Value::Null) => None,
10032        Some(value) => value_as_u64(value),
10033        None => Some(1),
10034    };
10035    let backoff_seconds = policy
10036        .and_then(|policy| policy.get("backoff_seconds"))
10037        .and_then(Value::as_array)
10038        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
10039        .unwrap_or_default();
10040    let non_retryable_error_types = policy
10041        .and_then(|policy| policy.get("non_retryable_error_types"))
10042        .and_then(Value::as_array)
10043        .into_iter()
10044        .flatten()
10045        .filter_map(Value::as_str)
10046        .map(str::to_string)
10047        .collect();
10048
10049    ActivityRetrySnapshot {
10050        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
10051        max_attempts: RecordedSnapshotValue::Known(max_attempts),
10052        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
10053        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
10054        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
10055        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
10056        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
10057        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
10058    }
10059}
10060
10061fn activity_options_description(options: &RecordedActivityOptions) -> String {
10062    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
10063}
10064
10065impl RecordedCommand {
10066    fn sequence(&self) -> u64 {
10067        match self {
10068            Self::Activity { sequence, .. }
10069            | Self::Timer { sequence, .. }
10070            | Self::ChildWorkflow { sequence, .. }
10071            | Self::SignalWait { sequence, .. }
10072            | Self::ConditionWait { sequence, .. }
10073            | Self::SearchAttributes { sequence, .. }
10074            | Self::SideEffect { sequence, .. }
10075            | Self::VersionMarker { sequence, .. }
10076            | Self::Memo { sequence, .. } => *sequence,
10077        }
10078    }
10079
10080    fn shape(&self) -> &'static str {
10081        match self {
10082            Self::Activity { .. } => "activity",
10083            Self::Timer { .. } => "timer",
10084            Self::ChildWorkflow { .. } => "child workflow",
10085            Self::SignalWait { .. } => "signal wait",
10086            Self::ConditionWait { .. } => "condition wait",
10087            Self::SearchAttributes { .. } => "search-attribute update",
10088            Self::SideEffect { .. } => "side effect",
10089            Self::VersionMarker { .. } => "version marker",
10090            Self::Memo { .. } => "memo upsert",
10091        }
10092    }
10093}
10094
10095fn ensure_version_supported(
10096    change_id: &str,
10097    version: i32,
10098    min_supported: i32,
10099    max_supported: i32,
10100    sequence: u64,
10101) -> Result<()> {
10102    if (min_supported..=max_supported).contains(&version) {
10103        return Ok(());
10104    }
10105    Err(Error::NonDeterministicReplay(ReplayFailure::new(
10106        "version_marker_incompatible_range",
10107        (sequence != 0).then_some(sequence),
10108        Some(format!("{min_supported}..={max_supported}")),
10109        Some(format!("{change_id}:{version}")),
10110        "recorded workflow version is outside the range supported by current code",
10111    )))
10112}
10113
10114#[derive(Clone, Debug)]
10115struct ResumeSignal {
10116    signal_name: String,
10117    arguments: Vec<AvroValue>,
10118}
10119
10120const MAX_PARALLEL_OPERATIONS: usize = 1000;
10121
10122fn parallel_group_prefix(kind: &str) -> &'static str {
10123    match kind {
10124        "activity" => "parallel-activities",
10125        "child" => "parallel-children",
10126        "timer" => "parallel-timers",
10127        _ => "parallel-calls",
10128    }
10129}
10130
10131fn parallel_group_entry(
10132    base_sequence: u64,
10133    size: usize,
10134    index: usize,
10135    kind: &str,
10136) -> ParallelGroupMetadata {
10137    ParallelGroupMetadata {
10138        parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
10139        parallel_group_kind: kind.to_string(),
10140        parallel_group_base_sequence: base_sequence,
10141        parallel_group_size: size,
10142        parallel_group_index: index,
10143        parallel_group_mode: None,
10144        selection_member_key: None,
10145        selection_member_index: None,
10146        selection_member_base_sequence: None,
10147        selection_member_size: None,
10148        selection_member_kind: None,
10149    }
10150}
10151
10152struct SelectionMemberMetadata {
10153    key: SelectionKey,
10154    index: usize,
10155    base_sequence: u64,
10156    size: usize,
10157    kind: String,
10158}
10159
10160fn selection_group_entry(
10161    base_sequence: u64,
10162    size: usize,
10163    index: usize,
10164    kind: &str,
10165    member: &SelectionMemberMetadata,
10166) -> ParallelGroupMetadata {
10167    ParallelGroupMetadata {
10168        parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
10169        parallel_group_kind: kind.to_string(),
10170        parallel_group_base_sequence: base_sequence,
10171        parallel_group_size: size,
10172        parallel_group_index: index,
10173        parallel_group_mode: Some("select".to_string()),
10174        selection_member_key: Some(member.key.clone()),
10175        selection_member_index: Some(member.index),
10176        selection_member_base_sequence: Some(member.base_sequence),
10177        selection_member_size: Some(member.size),
10178        selection_member_kind: Some(member.kind.clone()),
10179    }
10180}
10181
10182fn apply_parallel_group_path(
10183    command: &mut serde_json::Map<String, Value>,
10184    path: &[ParallelGroupMetadata],
10185) {
10186    let Some(inner) = path.last() else {
10187        return;
10188    };
10189    command.insert(
10190        "parallel_group_id".to_string(),
10191        json!(inner.parallel_group_id),
10192    );
10193    command.insert(
10194        "parallel_group_kind".to_string(),
10195        json!(inner.parallel_group_kind),
10196    );
10197    command.insert(
10198        "parallel_group_base_sequence".to_string(),
10199        json!(inner.parallel_group_base_sequence),
10200    );
10201    command.insert(
10202        "parallel_group_size".to_string(),
10203        json!(inner.parallel_group_size),
10204    );
10205    command.insert(
10206        "parallel_group_index".to_string(),
10207        json!(inner.parallel_group_index),
10208    );
10209    if let Some(mode) = &inner.parallel_group_mode {
10210        command.insert("parallel_group_mode".to_string(), json!(mode));
10211    }
10212    if let Some(key) = &inner.selection_member_key {
10213        command.insert("selection_member_key".to_string(), json!(key));
10214    }
10215    if let Some(index) = inner.selection_member_index {
10216        command.insert("selection_member_index".to_string(), json!(index));
10217    }
10218    if let Some(base_sequence) = inner.selection_member_base_sequence {
10219        command.insert(
10220            "selection_member_base_sequence".to_string(),
10221            json!(base_sequence),
10222        );
10223    }
10224    if let Some(size) = inner.selection_member_size {
10225        command.insert("selection_member_size".to_string(), json!(size));
10226    }
10227    if let Some(kind) = &inner.selection_member_kind {
10228        command.insert("selection_member_kind".to_string(), json!(kind));
10229    }
10230    command.insert("parallel_group_path".to_string(), json!(path));
10231}
10232
10233fn ensure_parallel_path_matches(
10234    sequence: u64,
10235    recorded: Option<&[ParallelGroupMetadata]>,
10236    expected: &[ParallelGroupMetadata],
10237) -> Result<()> {
10238    match (recorded, expected.is_empty()) {
10239        (None, true) => Ok(()),
10240        (Some(recorded), false) if recorded == expected => Ok(()),
10241        (None, false) => Err(invalid_recorded_history(
10242            "parallel_group_metadata_missing",
10243            sequence,
10244            &serde_json::to_string(expected).unwrap_or_default(),
10245            "<missing>",
10246            "recorded parallel member is missing its durable group path",
10247        )),
10248        (Some(recorded), true) => Err(invalid_recorded_history(
10249            "parallel_group_shape_mismatch",
10250            sequence,
10251            "sequential command",
10252            &serde_json::to_string(recorded).unwrap_or_default(),
10253            "recorded command belonged to a parallel group but current code schedules it sequentially",
10254        )),
10255        (Some(recorded), false) => Err(invalid_recorded_history(
10256            "parallel_group_shape_mismatch",
10257            sequence,
10258            &serde_json::to_string(recorded).unwrap_or_default(),
10259            &serde_json::to_string(expected).unwrap_or_default(),
10260            "recorded parallel-group identity or path changed during replay",
10261        )),
10262    }
10263}
10264
10265#[derive(Clone, Debug)]
10266enum ParallelShape {
10267    Leaf,
10268    Group(Vec<ParallelShape>),
10269}
10270
10271struct ParallelDescriptor {
10272    operation: ParallelOperation,
10273    offset: usize,
10274    member_path: Vec<usize>,
10275    group_path: Vec<ParallelGroupMetadata>,
10276}
10277
10278fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
10279    operations
10280        .iter()
10281        .map(|operation| match operation {
10282            ParallelOperation::Group(children) => parallel_leaf_count(children),
10283            _ => 1,
10284        })
10285        .sum()
10286}
10287
10288fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
10289    match operation {
10290        ParallelOperation::Activity { .. } => Some("activity"),
10291        ParallelOperation::ChildWorkflow { .. } => Some("child"),
10292        ParallelOperation::Timer(_) => Some("timer"),
10293        ParallelOperation::Signal(_) => Some("signal"),
10294        ParallelOperation::Condition { .. } => Some("condition"),
10295        ParallelOperation::Group(children) => parallel_group_kind(children),
10296    }
10297}
10298
10299fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
10300    let mut kind = None;
10301    for operation in operations {
10302        let Some(operation_kind) = parallel_operation_kind(operation) else {
10303            continue;
10304        };
10305        match kind {
10306            None => kind = Some(operation_kind),
10307            Some(current) if current == operation_kind => {}
10308            Some(_) => return Some("mixed"),
10309        }
10310    }
10311    kind
10312}
10313
10314fn validate_parallel_operations(
10315    operations: &[ParallelOperation],
10316    member_path: &mut Vec<usize>,
10317    root: bool,
10318) -> Result<()> {
10319    let leaves = parallel_leaf_count(operations);
10320    if leaves > MAX_PARALLEL_OPERATIONS {
10321        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10322            reason: "fan_out_limit_exceeded",
10323            member_path: member_path.clone(),
10324            message: format!(
10325                "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10326            ),
10327        }));
10328    }
10329    if !root && operations.is_empty() {
10330        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10331            reason: "nested_group_empty",
10332            member_path: member_path.clone(),
10333            message: "a nested group must contain at least one durable leaf".to_string(),
10334        }));
10335    }
10336
10337    for (index, operation) in operations.iter().enumerate() {
10338        member_path.push(index);
10339        match operation {
10340            ParallelOperation::Activity {
10341                options, arguments, ..
10342            } => {
10343                options
10344                    .validate()
10345                    .map_err(|error| Error::InvalidActivityOptions(error))?;
10346                if let Err(error) = arguments {
10347                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10348                        reason: "arguments_invalid",
10349                        member_path: member_path.clone(),
10350                        message: error.to_string(),
10351                    }));
10352                }
10353            }
10354            ParallelOperation::ChildWorkflow {
10355                options, arguments, ..
10356            } => {
10357                validate_parallel_child_options(options)?;
10358                if let Err(error) = arguments {
10359                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10360                        reason: "arguments_invalid",
10361                        member_path: member_path.clone(),
10362                        message: error.to_string(),
10363                    }));
10364                }
10365            }
10366            ParallelOperation::Timer(duration)
10367                if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10368            {
10369                return Err(Error::TimerDurationOverflow);
10370            }
10371            ParallelOperation::Timer(_) => {}
10372            ParallelOperation::Signal(signal_name) => {
10373                validate_user_signal_name(signal_name)?;
10374                if signal_name.trim().is_empty() {
10375                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10376                        reason: "signal_name_empty",
10377                        member_path: member_path.clone(),
10378                        message: "signal wait name must not be empty".to_string(),
10379                    }));
10380                }
10381            }
10382            ParallelOperation::Condition { options, .. } => {
10383                options.validate()?;
10384            }
10385            ParallelOperation::Group(children) => {
10386                validate_parallel_operations(children, member_path, false)?;
10387            }
10388        }
10389        member_path.pop();
10390    }
10391    Ok(())
10392}
10393
10394fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10395    if options.task_queue.trim().is_empty() {
10396        return Err(Error::InvalidChildWorkflowOptions(
10397            "task_queue must not be empty".to_string(),
10398        ));
10399    }
10400    for (name, value) in [
10401        (
10402            "execution_timeout_seconds",
10403            options.execution_timeout_seconds,
10404        ),
10405        ("run_timeout_seconds", options.run_timeout_seconds),
10406    ] {
10407        if value == Some(0) {
10408            return Err(Error::InvalidChildWorkflowOptions(format!(
10409                "{name} must be at least 1"
10410            )));
10411        }
10412    }
10413    if options
10414        .retry_policy
10415        .as_ref()
10416        .is_some_and(|policy| policy.max_attempts == Some(0))
10417    {
10418        return Err(Error::InvalidChildWorkflowOptions(
10419            "retry_policy.max_attempts must be at least 1".to_string(),
10420        ));
10421    }
10422    Ok(())
10423}
10424
10425fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10426    ParallelShape::Group(
10427        operations
10428            .iter()
10429            .map(|operation| match operation {
10430                ParallelOperation::Group(children) => parallel_shape(children),
10431                _ => ParallelShape::Leaf,
10432            })
10433            .collect(),
10434    )
10435}
10436
10437fn parallel_descriptors(
10438    operations: Vec<ParallelOperation>,
10439    base_sequence: u64,
10440) -> Result<Vec<ParallelDescriptor>> {
10441    let size = parallel_leaf_count(&operations);
10442    let kind = parallel_group_kind(&operations).unwrap_or("activity");
10443    let mut descriptors = Vec::with_capacity(size);
10444    let mut cursor = 0;
10445
10446    for (index, operation) in operations.into_iter().enumerate() {
10447        match operation {
10448            ParallelOperation::Group(children) => {
10449                let child_base = base_sequence
10450                    .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10451                    .ok_or(Error::TimerDurationOverflow)?;
10452                for mut descriptor in parallel_descriptors(children, child_base)? {
10453                    let outer_index = cursor + descriptor.offset;
10454                    descriptor.group_path.insert(
10455                        0,
10456                        parallel_group_entry(base_sequence, size, outer_index, kind),
10457                    );
10458                    descriptor.member_path.insert(0, index);
10459                    descriptor.offset = outer_index;
10460                    descriptors.push(descriptor);
10461                }
10462                cursor = descriptors.len();
10463            }
10464            operation => {
10465                descriptors.push(ParallelDescriptor {
10466                    operation,
10467                    offset: cursor,
10468                    member_path: vec![index],
10469                    group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10470                });
10471                cursor += 1;
10472            }
10473        }
10474    }
10475    Ok(descriptors)
10476}
10477
10478enum ParallelLeafCall {
10479    Activity(ActivityCall),
10480    ChildWorkflow(ChildWorkflowCall),
10481    Timer(TimerCall),
10482    Signal(SignalCall),
10483    Condition(ConditionWaitCall),
10484}
10485
10486fn parallel_leaf_call(
10487    ctx: &WorkflowContext,
10488    operation: ParallelOperation,
10489    parallel_group_path: Vec<ParallelGroupMetadata>,
10490) -> ParallelLeafCall {
10491    match operation {
10492        ParallelOperation::Activity {
10493            activity_type,
10494            options,
10495            arguments,
10496        } => ParallelLeafCall::Activity(ActivityCall {
10497            ctx: ctx.clone(),
10498            activity_type,
10499            options,
10500            args: Some(arguments),
10501            scheduled: false,
10502            parallel_group_path,
10503        }),
10504        ParallelOperation::ChildWorkflow {
10505            workflow_type,
10506            options,
10507            arguments,
10508        } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10509            ctx: ctx.clone(),
10510            workflow_type,
10511            options,
10512            args: Some(arguments),
10513            scheduled: false,
10514            matched_pending: false,
10515            parallel_group_path,
10516        }),
10517        ParallelOperation::Timer(duration) => {
10518            let delay_seconds = duration
10519                .as_secs()
10520                .checked_add(u64::from(duration.subsec_nanos() > 0));
10521            ParallelLeafCall::Timer(TimerCall {
10522                ctx: ctx.clone(),
10523                delay_seconds,
10524                scheduled: false,
10525                matched_pending: false,
10526                parallel_group_path,
10527            })
10528        }
10529        ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10530            ctx: ctx.clone(),
10531            signal_name,
10532            runtime_reserved_allowed: false,
10533            opened_wait: false,
10534            matched_pending: false,
10535            parallel_group_path,
10536        }),
10537        ParallelOperation::Condition { options, predicate } => {
10538            ParallelLeafCall::Condition(ConditionWaitCall {
10539                ctx: ctx.clone(),
10540                options,
10541                predicate,
10542                occurrence_id: None,
10543                opened_wait: false,
10544                parallel_group_path,
10545            })
10546        }
10547        ParallelOperation::Group(_) => {
10548            unreachable!("parallel descriptors contain only durable leaves")
10549        }
10550    }
10551}
10552
10553impl ParallelLeafCall {
10554    fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10555        match self {
10556            Self::Activity(call) => Pin::new(call)
10557                .poll_avro_value(cx)
10558                .map_ok(ParallelAvroResult::Activity),
10559            Self::ChildWorkflow(call) => Pin::new(call)
10560                .poll_avro_value(cx)
10561                .map_ok(ParallelAvroResult::ChildWorkflow),
10562            Self::Timer(call) => Pin::new(call)
10563                .poll(cx)
10564                .map_ok(|()| ParallelAvroResult::Timer),
10565            Self::Signal(call) => Pin::new(call)
10566                .poll_avro_value(cx)
10567                .map_ok(ParallelAvroResult::Signal),
10568            Self::Condition(call) => Pin::new(call)
10569                .poll(cx)
10570                .map_ok(ParallelAvroResult::Condition),
10571        }
10572    }
10573}
10574
10575struct ParallelLeaf {
10576    call: ParallelLeafCall,
10577    member_path: Vec<usize>,
10578    group_path: Vec<ParallelGroupMetadata>,
10579    result: Option<ParallelAvroResult>,
10580}
10581
10582/// Future returned by [`WorkflowContext::parallel`].
10583pub struct ParallelCall {
10584    ctx: WorkflowContext,
10585    operations: Option<Vec<ParallelOperation>>,
10586    shape: Option<ParallelShape>,
10587    leaves: Vec<ParallelLeaf>,
10588}
10589
10590impl ParallelCall {
10591    fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10592        Self {
10593            ctx,
10594            operations: Some(operations),
10595            shape: None,
10596            leaves: Vec::new(),
10597        }
10598    }
10599
10600    fn initialize(&mut self) -> Result<()> {
10601        let operations = self.operations.take().unwrap_or_default();
10602        validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10603        self.shape = Some(parallel_shape(&operations));
10604        if operations.is_empty() {
10605            return Ok(());
10606        }
10607
10608        let base_sequence = {
10609            let state = self
10610                .ctx
10611                .state
10612                .lock()
10613                .map_err(|_| Error::WorkflowStatePoisoned)?;
10614            if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10615                recorded.sequence()
10616            } else {
10617                let last = state
10618                    .recorded_commands
10619                    .last()
10620                    .map(RecordedCommand::sequence)
10621                    .unwrap_or(0);
10622                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10623                    .and_then(|sequence| sequence.checked_add(1))
10624                    .ok_or_else(|| {
10625                        Error::InvalidParallelGroup(ParallelGroupError {
10626                            reason: "sequence_overflow",
10627                            member_path: Vec::new(),
10628                            message: "parallel group sequence identity overflowed u64".to_string(),
10629                        })
10630                    })?
10631            }
10632        };
10633
10634        self.leaves = parallel_descriptors(operations, base_sequence)?
10635            .into_iter()
10636            .map(|descriptor| {
10637                let call = parallel_leaf_call(
10638                    &self.ctx,
10639                    descriptor.operation,
10640                    descriptor.group_path.clone(),
10641                );
10642                ParallelLeaf {
10643                    call,
10644                    member_path: descriptor.member_path,
10645                    group_path: descriptor.group_path,
10646                    result: None,
10647                }
10648            })
10649            .collect();
10650        Ok(())
10651    }
10652
10653    fn poll_avro_value(
10654        mut self: Pin<&mut Self>,
10655        cx: &mut TaskContext<'_>,
10656    ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10657        if self.operations.is_some() {
10658            if let Err(error) = self.initialize() {
10659                return Poll::Ready(Err(error));
10660            }
10661        }
10662        if self.leaves.is_empty() {
10663            return Poll::Ready(Ok(Vec::new()));
10664        }
10665
10666        let mut failures = Vec::new();
10667        let mut pending = false;
10668        for (index, leaf) in self.leaves.iter_mut().enumerate() {
10669            if leaf.result.is_some() {
10670                continue;
10671            }
10672            match leaf.call.poll_avro_value(cx) {
10673                Poll::Ready(Ok(result)) => leaf.result = Some(result),
10674                Poll::Ready(Err(error)) => failures.push((index, error)),
10675                Poll::Pending => pending = true,
10676            }
10677        }
10678
10679        if !failures.is_empty() {
10680            if let Some(position) = failures
10681                .iter()
10682                .position(|(_, error)| workflow_task_integrity_error(error))
10683            {
10684                return Poll::Ready(Err(failures.remove(position).1));
10685            }
10686            failures.sort_by_key(|(index, _)| *index);
10687            let (failed_index, cause) = failures.remove(0);
10688            let failed = &self.leaves[failed_index];
10689            let completed = self
10690                .leaves
10691                .iter()
10692                .filter_map(|leaf| {
10693                    leaf.result
10694                        .clone()
10695                        .and_then(|result| result.into_json_result().ok())
10696                        .map(|result| ParallelCompletion {
10697                            member_path: leaf.member_path.clone(),
10698                            result,
10699                        })
10700                })
10701                .collect();
10702            let group_id = failed
10703                .group_path
10704                .first()
10705                .map(|entry| entry.parallel_group_id.clone())
10706                .unwrap_or_default();
10707            return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10708                group_id,
10709                member_path: failed.member_path.clone(),
10710                group_path: failed.group_path.clone(),
10711                completed,
10712                cause: Box::new(cause),
10713            })));
10714        }
10715        if pending {
10716            return Poll::Pending;
10717        }
10718
10719        let mut flat_results = self
10720            .leaves
10721            .iter_mut()
10722            .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10723            .collect::<Vec<_>>()
10724            .into_iter();
10725        let results = parallel_results_for_shape(
10726            self.shape.as_ref().expect("initialized parallel shape"),
10727            &mut flat_results,
10728        );
10729        Poll::Ready(Ok(match results {
10730            ParallelAvroResult::Group(results) => results,
10731            ParallelAvroResult::Activity(_)
10732            | ParallelAvroResult::ChildWorkflow(_)
10733            | ParallelAvroResult::Timer
10734            | ParallelAvroResult::Signal(_)
10735            | ParallelAvroResult::Condition(_) => {
10736                unreachable!("root parallel shape is a group")
10737            }
10738        }))
10739    }
10740}
10741
10742fn parallel_results_for_shape(
10743    shape: &ParallelShape,
10744    flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10745) -> ParallelAvroResult {
10746    match shape {
10747        ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10748        ParallelShape::Group(children) => ParallelAvroResult::Group(
10749            children
10750                .iter()
10751                .map(|child| parallel_results_for_shape(child, flat_results))
10752                .collect(),
10753        ),
10754    }
10755}
10756
10757impl Future for ParallelCall {
10758    type Output = Result<Vec<ParallelResult>>;
10759
10760    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10761        self.poll_avro_value(cx)
10762            .map_ok(|results| {
10763                results
10764                    .into_iter()
10765                    .map(ParallelAvroResult::into_json_result)
10766                    .collect::<Result<Vec<_>>>()
10767            })
10768            .map_ok(|result| result)
10769            .flatten_result()
10770    }
10771}
10772
10773#[derive(Clone, Debug)]
10774struct SelectionMemberPlan {
10775    key: SelectionKey,
10776    index: usize,
10777    base_sequence: u64,
10778    size: usize,
10779    kind: String,
10780    shape: ParallelShape,
10781    leaf_start: usize,
10782}
10783
10784fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10785    match operation {
10786        ParallelOperation::Activity { .. } => "activity",
10787        ParallelOperation::ChildWorkflow { .. } => "child",
10788        ParallelOperation::Timer(_) => "timer",
10789        ParallelOperation::Signal(_) => "signal",
10790        ParallelOperation::Condition { .. } => "condition",
10791        ParallelOperation::Group(_) => "group",
10792    }
10793}
10794
10795fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10796    match operation {
10797        ParallelOperation::Group(children) => parallel_shape(children),
10798        _ => ParallelShape::Leaf,
10799    }
10800}
10801
10802fn selection_descriptors(
10803    operations: Vec<(SelectionKey, ParallelOperation)>,
10804    base_sequence: u64,
10805) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10806    if operations.is_empty() {
10807        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10808            reason: "selection_empty",
10809            member_path: Vec::new(),
10810            message: "durable selection requires at least one operation".to_string(),
10811        }));
10812    }
10813    let operation_refs = operations
10814        .iter()
10815        .map(|(_, operation)| operation)
10816        .collect::<Vec<_>>();
10817    let total_size = operation_refs
10818        .iter()
10819        .map(|operation| match operation {
10820            ParallelOperation::Group(children) => parallel_leaf_count(children),
10821            _ => 1,
10822        })
10823        .sum::<usize>();
10824    if total_size > MAX_PARALLEL_OPERATIONS {
10825        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10826            reason: "fan_out_limit_exceeded",
10827            member_path: Vec::new(),
10828            message: format!(
10829                "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10830            ),
10831        }));
10832    }
10833    let group_kind = {
10834        let mut kind = None;
10835        for operation in &operation_refs {
10836            let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10837            match kind {
10838                None => kind = Some(operation_kind),
10839                Some(current) if current == operation_kind => {}
10840                Some(_) => {
10841                    kind = Some("mixed");
10842                    break;
10843                }
10844            }
10845        }
10846        kind.unwrap_or("mixed")
10847    };
10848
10849    let mut descriptors = Vec::with_capacity(total_size);
10850    let mut members = Vec::with_capacity(operations.len());
10851    let mut cursor = 0usize;
10852    let mut seen_keys: Vec<SelectionKey> = Vec::new();
10853    for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10854        if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10855            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10856                reason: "selection_key_invalid",
10857                member_path: vec![member_index],
10858                message: "selection member keys must be non-empty strings or non-negative integers"
10859                    .to_string(),
10860            }));
10861        }
10862        if seen_keys.contains(&key) {
10863            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10864                reason: "selection_key_duplicate",
10865                member_path: vec![member_index],
10866                message: format!("selection member key {key:?} is duplicated"),
10867            }));
10868        }
10869        seen_keys.push(key.clone());
10870        let member_size = match &operation {
10871            ParallelOperation::Group(children) => parallel_leaf_count(children),
10872            _ => 1,
10873        };
10874        if member_size == 0 {
10875            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10876                reason: "selection_member_empty",
10877                member_path: vec![member_index],
10878                message: "a selection member must contain at least one durable leaf".to_string(),
10879            }));
10880        }
10881        let member_base = base_sequence
10882            .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10883            .ok_or(Error::TimerDurationOverflow)?;
10884        let member_kind = selection_operation_kind(&operation).to_string();
10885        let member_shape = selection_operation_shape(&operation);
10886        let leaf_start = descriptors.len();
10887        match operation {
10888            ParallelOperation::Group(children) => {
10889                validate_parallel_operations(&children, &mut vec![member_index], false)?;
10890                for mut descriptor in parallel_descriptors(children, member_base)? {
10891                    let flat_index = cursor + descriptor.offset;
10892                    descriptor.group_path.insert(
10893                        0,
10894                        selection_group_entry(
10895                            base_sequence,
10896                            total_size,
10897                            flat_index,
10898                            group_kind,
10899                            &SelectionMemberMetadata {
10900                                key: key.clone(),
10901                                index: member_index,
10902                                base_sequence: member_base,
10903                                size: member_size,
10904                                kind: member_kind.clone(),
10905                            },
10906                        ),
10907                    );
10908                    descriptor.member_path.insert(0, member_index);
10909                    descriptor.offset = flat_index;
10910                    descriptors.push(descriptor);
10911                }
10912            }
10913            operation => {
10914                validate_parallel_operations(
10915                    std::slice::from_ref(&operation),
10916                    &mut Vec::new(),
10917                    true,
10918                )?;
10919                descriptors.push(ParallelDescriptor {
10920                    operation,
10921                    offset: cursor,
10922                    member_path: vec![member_index],
10923                    group_path: vec![selection_group_entry(
10924                        base_sequence,
10925                        total_size,
10926                        cursor,
10927                        group_kind,
10928                        &SelectionMemberMetadata {
10929                            key: key.clone(),
10930                            index: member_index,
10931                            base_sequence: member_base,
10932                            size: member_size,
10933                            kind: member_kind.clone(),
10934                        },
10935                    )],
10936                });
10937            }
10938        }
10939        members.push(SelectionMemberPlan {
10940            key,
10941            index: member_index,
10942            base_sequence: member_base,
10943            size: member_size,
10944            kind: member_kind,
10945            shape: member_shape,
10946            leaf_start,
10947        });
10948        cursor += member_size;
10949    }
10950    Ok((descriptors, members))
10951}
10952
10953struct SelectionLeaf {
10954    call: ParallelLeafCall,
10955    outcome: Option<Result<ParallelAvroResult>>,
10956}
10957
10958/// Stable reference to one member of a durable selection group.
10959#[derive(Clone)]
10960pub struct DurableOperationHandle {
10961    ctx: WorkflowContext,
10962    pub key: SelectionKey,
10963    pub index: usize,
10964    pub kind: String,
10965    pub identity: String,
10966    pub base_sequence: u64,
10967    pub size: usize,
10968    pub selection_group_id: String,
10969    shape: ParallelShape,
10970}
10971
10972impl std::fmt::Debug for DurableOperationHandle {
10973    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10974        formatter
10975            .debug_struct("DurableOperationHandle")
10976            .field("key", &self.key)
10977            .field("index", &self.index)
10978            .field("kind", &self.kind)
10979            .field("identity", &self.identity)
10980            .field("base_sequence", &self.base_sequence)
10981            .field("size", &self.size)
10982            .field("selection_group_id", &self.selection_group_id)
10983            .finish()
10984    }
10985}
10986
10987impl DurableOperationHandle {
10988    /// Await this member after another member has already won the selection.
10989    pub fn await_result(&self) -> DurableOperationAwaitCall {
10990        DurableOperationAwaitCall {
10991            handle: self.clone(),
10992        }
10993    }
10994
10995    /// Request cancellation without affecting siblings. The future resolves
10996    /// to unit; only `SelectionOperationCancelled` history proves cancellation
10997    /// beat a concurrently committed terminal result.
10998    pub fn cancel(&self) -> CancelDurableOperationCall {
10999        CancelDurableOperationCall {
11000            handle: self.clone(),
11001            emitted: false,
11002        }
11003    }
11004}
11005
11006/// The one winner committed for a durable selection group.
11007#[derive(Debug)]
11008pub struct SelectionResult {
11009    pub key: SelectionKey,
11010    pub index: usize,
11011    pub kind: String,
11012    pub identity: String,
11013    pub value: Option<ParallelResult>,
11014    pub failure: Option<Error>,
11015    pub winner: DurableOperationHandle,
11016    pub handles: Vec<DurableOperationHandle>,
11017}
11018
11019impl SelectionResult {
11020    pub fn succeeded(&self) -> bool {
11021        self.failure.is_none()
11022    }
11023
11024    pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
11025        self.handles.iter().find(|handle| &handle.key == key)
11026    }
11027
11028    pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
11029        self.handles
11030            .iter()
11031            .filter(|handle| handle.index != self.index)
11032            .collect()
11033    }
11034
11035    pub fn into_result(self) -> Result<ParallelResult> {
11036        match (self.value, self.failure) {
11037            (Some(value), None) => Ok(value),
11038            (_, Some(error)) => Err(error),
11039            _ => Err(Error::WorkerLoop(
11040                "selection result contained neither a value nor a failure".to_string(),
11041            )),
11042        }
11043    }
11044}
11045
11046/// Future returned by [`WorkflowContext::select`].
11047pub struct SelectCall {
11048    ctx: WorkflowContext,
11049    operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
11050    members: Vec<SelectionMemberPlan>,
11051    leaves: Vec<SelectionLeaf>,
11052    group_id: Option<String>,
11053}
11054
11055impl SelectCall {
11056    fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
11057        Self {
11058            ctx,
11059            operations: Some(operations),
11060            members: Vec::new(),
11061            leaves: Vec::new(),
11062            group_id: None,
11063        }
11064    }
11065
11066    fn initialize(&mut self) -> Result<()> {
11067        let operations = self.operations.take().unwrap_or_default();
11068        let base_sequence = {
11069            let state = self
11070                .ctx
11071                .state
11072                .lock()
11073                .map_err(|_| Error::WorkflowStatePoisoned)?;
11074            if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
11075                marker.selection_group_base_sequence
11076            } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
11077                recorded.sequence()
11078            } else {
11079                let last = state
11080                    .recorded_commands
11081                    .last()
11082                    .map(RecordedCommand::sequence)
11083                    .unwrap_or(0);
11084                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
11085                    .and_then(|sequence| sequence.checked_add(1))
11086                    .ok_or(Error::TimerDurationOverflow)?
11087            }
11088        };
11089        let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
11090        let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
11091        self.leaves = descriptors
11092            .into_iter()
11093            .map(|descriptor| SelectionLeaf {
11094                call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
11095                outcome: None,
11096            })
11097            .collect();
11098        self.members = members;
11099        self.group_id = Some(group_id);
11100        Ok(())
11101    }
11102}
11103
11104impl Future for SelectCall {
11105    type Output = Result<SelectionResult>;
11106
11107    fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11108        if self.operations.is_some() {
11109            if let Err(error) = self.initialize() {
11110                return Poll::Ready(Err(error));
11111            }
11112        }
11113
11114        for leaf in &mut self.leaves {
11115            if leaf.outcome.is_some() {
11116                continue;
11117            }
11118            if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
11119                if outcome
11120                    .as_ref()
11121                    .err()
11122                    .is_some_and(workflow_task_integrity_error)
11123                {
11124                    return Poll::Ready(outcome.map(|_| unreachable!()));
11125                }
11126                leaf.outcome = Some(outcome);
11127            }
11128        }
11129
11130        let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
11131        let selection_member_range = self
11132            .members
11133            .first()
11134            .map(|member| member.base_sequence)
11135            .zip(self.leaves.len().try_into().ok())
11136            .map(|(base_sequence, size): (u64, u64)| {
11137                base_sequence..base_sequence.saturating_add(size)
11138            });
11139        let marker = {
11140            let mut state = match self.ctx.state.lock() {
11141                Ok(state) => state,
11142                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11143            };
11144            let marker = state
11145                .selection_markers
11146                .get(state.selection_marker_cursor)
11147                .cloned();
11148            if marker.is_none()
11149                && all_members_terminal
11150                && selection_member_range.as_ref().is_some_and(|member_range| {
11151                    state
11152                        .recorded_commands
11153                        .iter()
11154                        .any(|command| member_range.contains(&command.sequence()))
11155                })
11156            {
11157                // Terminal member history can be committed before the server's
11158                // SelectionResolved marker becomes visible to this task. That
11159                // marker is the durable barrier the selection is still waiting
11160                // on, so an otherwise commandless replay remains legitimately
11161                // pending instead of failing as an untracked yield.
11162                state.matched_recorded_pending = true;
11163            }
11164            marker
11165        };
11166        let Some(marker) = marker else {
11167            return Poll::Pending;
11168        };
11169        if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
11170            || marker.selection_group_size != self.leaves.len()
11171            || self.members.first().map(|member| member.base_sequence)
11172                != Some(marker.selection_group_base_sequence)
11173        {
11174            return Poll::Ready(Err(invalid_recorded_history(
11175                "selection_group_shape_mismatch",
11176                marker.selection_group_base_sequence,
11177                self.group_id
11178                    .as_deref()
11179                    .unwrap_or("initialized selection group"),
11180                &marker.selection_group_id,
11181                "recorded selection group differs from current workflow code",
11182            )));
11183        }
11184        let Some(member_position) = self.members.iter().position(|member| {
11185            member.key == marker.member_key
11186                && member.index == marker.member_index
11187                && member.base_sequence == marker.member_base_sequence
11188                && member.size == marker.member_size
11189                && member.kind == marker.operation_kind
11190        }) else {
11191            return Poll::Ready(Err(invalid_recorded_history(
11192                "selection_member_shape_mismatch",
11193                marker.member_base_sequence,
11194                "winner member matching current workflow code",
11195                &format!("{:?}", marker.member_key),
11196                "recorded selection winner differs from the authored member identity",
11197            )));
11198        };
11199        let member = self.members[member_position].clone();
11200        let (handles, resolution_sequence) = {
11201            let mut state = match self.ctx.state.lock() {
11202                Ok(state) => state,
11203                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11204            };
11205            let identities = self
11206                .members
11207                .iter()
11208                .map(|candidate| {
11209                    selection_operation_identity(
11210                        &state,
11211                        &candidate.kind,
11212                        candidate.base_sequence,
11213                        candidate.size,
11214                    )
11215                })
11216                .collect::<Vec<_>>();
11217            if let Some((position, missing)) = identities
11218                .iter()
11219                .enumerate()
11220                .find(|(_, identity)| identity.is_empty())
11221                .map(|(position, identity)| (position, identity.clone()))
11222            {
11223                let candidate = &self.members[position];
11224                return Poll::Ready(Err(invalid_recorded_history(
11225                    "selection_operation_identity_missing",
11226                    candidate.base_sequence,
11227                    &format!(
11228                        "durable {} resource identity from scheduled/open history",
11229                        candidate.kind
11230                    ),
11231                    &missing,
11232                    "selection member history is missing its canonical durable identity",
11233                )));
11234            }
11235            let expected_winner_identity = &identities[member_position];
11236            let resolution_sequence = match validated_selection_resolution_sequence(
11237                &state,
11238                &marker,
11239                &member,
11240                expected_winner_identity,
11241            ) {
11242                Ok(sequence) => sequence,
11243                Err(error) => return Poll::Ready(Err(error)),
11244            };
11245            let handles = self
11246                .members
11247                .iter()
11248                .zip(identities)
11249                .map(|(member, identity)| DurableOperationHandle {
11250                    ctx: self.ctx.clone(),
11251                    key: member.key.clone(),
11252                    index: member.index,
11253                    kind: member.kind.clone(),
11254                    identity,
11255                    base_sequence: member.base_sequence,
11256                    size: member.size,
11257                    selection_group_id: marker.selection_group_id.clone(),
11258                    shape: member.shape.clone(),
11259                })
11260                .collect::<Vec<_>>();
11261            if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
11262                return Poll::Ready(Err(error));
11263            }
11264            state.selection_marker_cursor += 1;
11265            (handles, resolution_sequence)
11266        };
11267
11268        let mut winner_failure = None;
11269        let mut flat_results = Vec::with_capacity(member.size);
11270        if marker.outcome == "failed" {
11271            let resolution_offset = match resolution_sequence
11272                .checked_sub(member.base_sequence)
11273                .and_then(|offset| usize::try_from(offset).ok())
11274            {
11275                Some(offset) if offset < member.size => offset,
11276                _ => {
11277                    return Poll::Ready(Err(invalid_recorded_history(
11278                        "selection_resolution_event_mismatch",
11279                        member.base_sequence,
11280                        "failure event within selected member bounds",
11281                        &resolution_sequence.to_string(),
11282                        "selection failure event is outside the authored member",
11283                    )))
11284                }
11285            };
11286            let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
11287            match leaf.outcome.take() {
11288                Some(Err(error)) => winner_failure = Some(error),
11289                _ => {
11290                    return Poll::Ready(Err(invalid_recorded_history(
11291                        "selection_winner_outcome_mismatch",
11292                        member.base_sequence,
11293                        "exact failed terminal history referenced by SelectionResolved",
11294                        "missing or successful resolution event",
11295                        "selection winner marker disagrees with terminal operation history",
11296                    )))
11297                }
11298            }
11299        } else {
11300            for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
11301                match leaf.outcome.take() {
11302                    Some(Ok(result)) => flat_results.push(result),
11303                    Some(Err(_)) => {
11304                        return Poll::Ready(Err(invalid_recorded_history(
11305                            "selection_winner_outcome_mismatch",
11306                            member.base_sequence,
11307                            "fully completed nested selection member",
11308                            "failed durable leaf",
11309                            "completed selection winner contains a failed leaf",
11310                        )))
11311                    }
11312                    None => {
11313                        return Poll::Ready(Err(invalid_recorded_history(
11314                            "selection_winner_unresolved",
11315                            member.base_sequence,
11316                            "terminal history for every completed winner leaf",
11317                            "pending member history",
11318                            "completed SelectionResolved member has an unfinished durable barrier",
11319                        )))
11320                    }
11321                }
11322            }
11323        }
11324        let value = if winner_failure.is_none() {
11325            let mut flat_results = flat_results.into_iter();
11326            let value = parallel_results_for_shape(&member.shape, &mut flat_results);
11327            match value.into_json_result() {
11328                Ok(value) => Some(value),
11329                Err(error) => return Poll::Ready(Err(error)),
11330            }
11331        } else {
11332            None
11333        };
11334        let winner = handles[member_position].clone();
11335        Poll::Ready(Ok(SelectionResult {
11336            key: winner.key.clone(),
11337            index: winner.index,
11338            kind: winner.kind.clone(),
11339            identity: winner.identity.clone(),
11340            value,
11341            failure: winner_failure,
11342            winner,
11343            handles,
11344        }))
11345    }
11346}
11347
11348fn selection_operation_identity(
11349    state: &WorkflowState,
11350    kind: &str,
11351    base_sequence: u64,
11352    size: usize,
11353) -> String {
11354    if kind == "group" {
11355        return format!("group:{base_sequence}:{size}");
11356    }
11357    let fields: &[&str] = match kind {
11358        "activity" => &["activity_execution_id"],
11359        "child" => &["child_workflow_run_id"],
11360        "timer" => &["timer_id"],
11361        "signal" => &["signal_wait_id"],
11362        "condition" => &["condition_wait_id"],
11363        _ => &[],
11364    };
11365    for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11366        for event in state
11367            .history_events
11368            .iter()
11369            .filter(|event| durable_event_sequence(event) == Some(sequence))
11370        {
11371            for field in fields {
11372                if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11373                    if !identity.is_empty() {
11374                        return identity.to_string();
11375                    }
11376                }
11377            }
11378        }
11379    }
11380    String::new()
11381}
11382
11383fn validated_selection_resolution_sequence(
11384    state: &WorkflowState,
11385    marker: &SelectionMarker,
11386    member: &SelectionMemberPlan,
11387    expected_identity: &str,
11388) -> Result<u64> {
11389    if expected_identity.is_empty() {
11390        return Err(invalid_recorded_history(
11391            "selection_operation_identity_missing",
11392            member.base_sequence,
11393            &format!(
11394                "durable {} resource identity from scheduled/open history",
11395                member.kind
11396            ),
11397            "missing operation identity",
11398            "selection member history is missing its canonical durable identity",
11399        ));
11400    }
11401    if marker.operation_identity != expected_identity {
11402        return Err(invalid_recorded_history(
11403            "selection_operation_identity_mismatch",
11404            member.base_sequence,
11405            expected_identity,
11406            &marker.operation_identity,
11407            "selection winner identity does not match durable scheduled/open history",
11408        ));
11409    }
11410
11411    let failure_types = [
11412        "ActivityFailed",
11413        "ActivityCancelled",
11414        "ActivityTimedOut",
11415        "ChildRunFailed",
11416        "ChildRunCancelled",
11417        "ChildRunTerminated",
11418    ];
11419    let success_types = [
11420        "ActivityCompleted",
11421        "ChildRunCompleted",
11422        "TimerFired",
11423        "SignalApplied",
11424        "ConditionWaitSatisfied",
11425        "ConditionWaitTimedOut",
11426    ];
11427    let terminal_types: &[&str] = if marker.outcome == "failed" {
11428        &failure_types
11429    } else {
11430        &success_types
11431    };
11432    let mut candidates = Vec::new();
11433    for event in state.history_events.iter() {
11434        let Some(sequence) = durable_event_sequence(event) else {
11435            continue;
11436        };
11437        if sequence < member.base_sequence
11438            || sequence >= member.base_sequence.saturating_add(member.size as u64)
11439            || !terminal_types.contains(&event.event_type.as_str())
11440        {
11441            continue;
11442        }
11443        let event_id = event
11444            .raw
11445            .get("id")
11446            .or_else(|| event.raw.get("event_id"))
11447            .and_then(Value::as_str)
11448            .filter(|value| !value.is_empty())
11449            .ok_or_else(|| {
11450                invalid_recorded_history(
11451                    "selection_resolution_event_id_missing",
11452                    member.base_sequence,
11453                    "terminal selection history with a durable event id",
11454                    &event.payload.to_string(),
11455                    "selection terminal history cannot be bound to its winner marker",
11456                )
11457            })?;
11458        candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11459    }
11460    let resolution = if marker.outcome == "failed" {
11461        candidates.first()
11462    } else {
11463        candidates.last()
11464    };
11465    let Some((event_id, event_type, sequence)) = resolution else {
11466        return Err(invalid_recorded_history(
11467            "selection_resolution_event_missing",
11468            member.base_sequence,
11469            "terminal history for the selected member",
11470            &format!("{:?}", marker.member_key),
11471            "selection winner marker has no matching durable terminal event",
11472        ));
11473    };
11474    if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11475        return Err(invalid_recorded_history(
11476            "selection_resolution_event_mismatch",
11477            member.base_sequence,
11478            &format!("{event_type}:{event_id}"),
11479            &format!(
11480                "{}:{}",
11481                marker.resolution_event_type, marker.resolution_event_id
11482            ),
11483            "selection winner marker does not reference the event that made its member terminal",
11484        ));
11485    }
11486    Ok(*sequence)
11487}
11488
11489fn recorded_selection_member_outcome(
11490    state: &WorkflowState,
11491    handle: &DurableOperationHandle,
11492) -> Result<Option<ParallelResult>> {
11493    for event in state.history_events.iter() {
11494        let Some(sequence) = durable_event_sequence(event) else {
11495            continue;
11496        };
11497        if sequence < handle.base_sequence
11498            || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11499            || !matches!(
11500                event.event_type.as_str(),
11501                "ActivityFailed"
11502                    | "ActivityCancelled"
11503                    | "ActivityTimedOut"
11504                    | "ChildRunFailed"
11505                    | "ChildRunCancelled"
11506                    | "ChildRunTerminated"
11507            )
11508        {
11509            continue;
11510        }
11511        let Some(command) = state
11512            .recorded_commands
11513            .iter()
11514            .find(|command| command.sequence() == sequence)
11515        else {
11516            continue;
11517        };
11518        match command {
11519            RecordedCommand::Activity {
11520                outcome: Some(Err(failure)),
11521                ..
11522            } => return Err(Error::ActivityFailed(failure.clone())),
11523            RecordedCommand::ChildWorkflow {
11524                outcome: Some(Err(failure)),
11525                ..
11526            } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11527            _ => {}
11528        }
11529    }
11530
11531    let mut results = Vec::with_capacity(handle.size);
11532    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11533        let Some(command) = state
11534            .recorded_commands
11535            .iter()
11536            .find(|command| command.sequence() == sequence)
11537        else {
11538            return Ok(None);
11539        };
11540        let result = match command {
11541            RecordedCommand::Activity { outcome, .. } => match outcome {
11542                Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11543                Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11544                None => return Ok(None),
11545            },
11546            RecordedCommand::Timer { fired, .. } => {
11547                if !fired {
11548                    return Ok(None);
11549                }
11550                ParallelAvroResult::Timer
11551            }
11552            RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11553                Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11554                Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11555                None => return Ok(None),
11556            },
11557            RecordedCommand::SignalWait { value, .. } => match value {
11558                Some(value) => ParallelAvroResult::Signal(value.clone()),
11559                None => return Ok(None),
11560            },
11561            RecordedCommand::ConditionWait { result, .. } => match result {
11562                Some(result) => ParallelAvroResult::Condition(*result),
11563                None => return Ok(None),
11564            },
11565            other => {
11566                return Err(command_mismatch(
11567                    other,
11568                    format!("selected {} member", handle.kind),
11569                ))
11570            }
11571        };
11572        results.push(result);
11573    }
11574    let mut results = results.into_iter();
11575    parallel_results_for_shape(&handle.shape, &mut results)
11576        .into_json_result()
11577        .map(Some)
11578}
11579
11580fn recorded_selection_member_is_terminal(
11581    state: &WorkflowState,
11582    handle: &DurableOperationHandle,
11583) -> bool {
11584    let mut completed = 0usize;
11585    let mut all_completed = true;
11586    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11587        let Some(command) = state
11588            .recorded_commands
11589            .iter()
11590            .find(|command| command.sequence() == sequence)
11591        else {
11592            all_completed = false;
11593            continue;
11594        };
11595        let terminal = match command {
11596            RecordedCommand::Activity {
11597                outcome: Some(Err(_)),
11598                ..
11599            }
11600            | RecordedCommand::ChildWorkflow {
11601                outcome: Some(Err(_)),
11602                ..
11603            } => return true,
11604            RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11605            RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11606            RecordedCommand::Timer { fired, .. } => *fired,
11607            RecordedCommand::SignalWait { value, .. } => value.is_some(),
11608            RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11609            RecordedCommand::SearchAttributes { .. }
11610            | RecordedCommand::SideEffect { .. }
11611            | RecordedCommand::VersionMarker { .. }
11612            | RecordedCommand::Memo { .. } => false,
11613        };
11614        if !terminal {
11615            all_completed = false;
11616            continue;
11617        }
11618        completed += 1;
11619    }
11620    all_completed && completed == handle.size
11621}
11622
11623fn selection_cancellation_for_handle(
11624    state: &WorkflowState,
11625    handle: &DurableOperationHandle,
11626) -> Result<bool> {
11627    let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11628        recorded.selection_group_id == handle.selection_group_id
11629            && recorded.member_base_sequence == handle.base_sequence
11630    }) else {
11631        return Ok(false);
11632    };
11633    validate_selection_cancellation_marker(marker, handle)?;
11634    Ok(true)
11635}
11636
11637fn validate_selection_cancellations_for_handles(
11638    state: &WorkflowState,
11639    handles: &[DurableOperationHandle],
11640) -> Result<()> {
11641    let Some(group_id) = handles
11642        .first()
11643        .map(|handle| handle.selection_group_id.as_str())
11644    else {
11645        return Ok(());
11646    };
11647    for marker in state
11648        .cancelled_selection_members
11649        .iter()
11650        .filter(|marker| marker.selection_group_id == group_id)
11651    {
11652        let Some(handle) = handles
11653            .iter()
11654            .find(|handle| handle.base_sequence == marker.member_base_sequence)
11655        else {
11656            return Err(invalid_recorded_history(
11657                "selection_cancellation_member_mismatch",
11658                marker.member_base_sequence,
11659                "SelectionOperationCancelled matching an authored selection handle",
11660                &format!("{marker:?}"),
11661                "selection cancellation member base does not name an authored member",
11662            ));
11663        };
11664        validate_selection_cancellation_marker(marker, handle)?;
11665    }
11666    Ok(())
11667}
11668
11669fn validate_selection_cancellation_marker(
11670    marker: &SelectionCancellation,
11671    handle: &DurableOperationHandle,
11672) -> Result<()> {
11673    if marker.selection_group_id != handle.selection_group_id
11674        || marker.member_key != handle.key
11675        || marker.member_index != handle.index
11676        || marker.member_base_sequence != handle.base_sequence
11677        || marker.member_size != handle.size
11678        || marker.operation_kind != handle.kind
11679        || marker.operation_identity != handle.identity
11680    {
11681        return Err(invalid_recorded_history(
11682            "selection_cancellation_member_mismatch",
11683            handle.base_sequence,
11684            "SelectionOperationCancelled matching the authored selection handle",
11685            &format!("{marker:?}"),
11686            "selection cancellation history targets different authored member metadata",
11687        ));
11688    }
11689    Ok(())
11690}
11691
11692/// Future returned by [`DurableOperationHandle::await_result`].
11693pub struct DurableOperationAwaitCall {
11694    handle: DurableOperationHandle,
11695}
11696
11697impl Future for DurableOperationAwaitCall {
11698    type Output = Result<ParallelResult>;
11699
11700    fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11701        let state = match self.handle.ctx.state.lock() {
11702            Ok(state) => state,
11703            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11704        };
11705        match selection_cancellation_for_handle(&state, &self.handle) {
11706            Err(error) => return Poll::Ready(Err(error)),
11707            Ok(false) => {}
11708            Ok(true) => {
11709                return Poll::Ready(Err(Error::DurableOperationCancelled(
11710                    DurableOperationCancelled {
11711                        selection_group_id: self.handle.selection_group_id.clone(),
11712                        member_key: self.handle.key.clone(),
11713                        member_index: self.handle.index,
11714                        operation_kind: self.handle.kind.clone(),
11715                        operation_identity: self.handle.identity.clone(),
11716                    },
11717                )));
11718            }
11719        }
11720        match recorded_selection_member_outcome(&state, &self.handle) {
11721            Ok(Some(result)) => Poll::Ready(Ok(result)),
11722            Ok(None) => Poll::Pending,
11723            Err(error) => Poll::Ready(Err(error)),
11724        }
11725    }
11726}
11727
11728/// Future returned by [`DurableOperationHandle::cancel`].
11729pub struct CancelDurableOperationCall {
11730    handle: DurableOperationHandle,
11731    emitted: bool,
11732}
11733
11734impl Future for CancelDurableOperationCall {
11735    type Output = Result<()>;
11736
11737    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11738        let ctx = self.handle.ctx.clone();
11739        let mut state = match ctx.state.lock() {
11740            Ok(state) => state,
11741            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11742        };
11743        match selection_cancellation_for_handle(&state, &self.handle) {
11744            Err(error) => return Poll::Ready(Err(error)),
11745            Ok(true) => return Poll::Ready(Ok(())),
11746            Ok(false) => {}
11747        }
11748        if recorded_selection_member_is_terminal(&state, &self.handle) {
11749            return Poll::Ready(Ok(()));
11750        }
11751        if !self.emitted {
11752            state.commands.push(json!({
11753                "type": "cancel_selection_operation",
11754                "selection_group_id": self.handle.selection_group_id,
11755                "member_key": self.handle.key,
11756                "member_index": self.handle.index,
11757                "member_base_sequence": self.handle.base_sequence,
11758                "member_size": self.handle.size,
11759                "operation_kind": self.handle.kind,
11760                "operation_identity": self.handle.identity,
11761            }));
11762            self.emitted = true;
11763        }
11764        // The cancellation command is only a request. Do not expose workflow
11765        // state after cancel until committed SelectionOperationCancelled
11766        // history proves that the request won the race with member completion.
11767        Poll::Pending
11768    }
11769}
11770
11771trait PollNestedResultExt<T> {
11772    fn flatten_result(self) -> Poll<Result<T>>;
11773}
11774
11775impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11776    fn flatten_result(self) -> Poll<Result<T>> {
11777        match self {
11778            Poll::Ready(Ok(result)) => Poll::Ready(result),
11779            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11780            Poll::Pending => Poll::Pending,
11781        }
11782    }
11783}
11784
11785struct SagaCompensation {
11786    activity_type: String,
11787    options: ActivityOptions,
11788    arguments: AvroValue,
11789    registration_order: usize,
11790}
11791
11792/// Workflow-local deterministic saga compensation helper.
11793///
11794/// Register each compensation only after its forward step succeeds. Passing
11795/// the forward `Result` to [`Saga::finish`] runs compensations sequentially in
11796/// reverse registration order after any failure, including cooperative
11797/// cancellation. Each compensation is an ordinary durable activity, so replay,
11798/// duplicate delivery, and worker restart use existing history semantics.
11799pub struct Saga {
11800    ctx: WorkflowContext,
11801    compensations: Vec<SagaCompensation>,
11802}
11803
11804impl Saga {
11805    fn new(ctx: WorkflowContext) -> Self {
11806        Self {
11807            ctx,
11808            compensations: Vec::new(),
11809        }
11810    }
11811
11812    pub fn add_compensation<T: Serialize>(
11813        &mut self,
11814        activity_type: impl Into<String>,
11815        args: T,
11816    ) -> Result<&mut Self> {
11817        self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11818    }
11819
11820    pub fn add_compensation_with_options<T: Serialize>(
11821        &mut self,
11822        activity_type: impl Into<String>,
11823        options: ActivityOptions,
11824        args: T,
11825    ) -> Result<&mut Self> {
11826        let activity_type = activity_type.into();
11827        if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11828            return Err(Error::Codec(
11829                "saga compensation activity type must be non-empty without surrounding whitespace"
11830                    .to_string(),
11831            ));
11832        }
11833        options.validate().map_err(Error::InvalidActivityOptions)?;
11834        let arguments = AvroValue::from_serialize(&args)?;
11835        let registration_order = self.compensations.len() + 1;
11836        self.compensations.push(SagaCompensation {
11837            activity_type,
11838            options,
11839            arguments,
11840            registration_order,
11841        });
11842        Ok(self)
11843    }
11844
11845    /// Compensate `initiating_failure` and return the failure that must remain.
11846    pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11847        while let Some(compensation) = self.compensations.pop() {
11848            if let Err(compensation_failure) = self
11849                .ctx
11850                .activity_with_options(
11851                    compensation.activity_type.clone(),
11852                    compensation.options,
11853                    compensation.arguments,
11854                )
11855                .await
11856            {
11857                if workflow_task_integrity_error(&compensation_failure) {
11858                    return compensation_failure;
11859                }
11860                return Error::SagaCompensationFailed(SagaCompensationFailure {
11861                    initiating_failure: Box::new(initiating_failure),
11862                    compensation_failure: Box::new(compensation_failure),
11863                    compensation_activity_type: compensation.activity_type,
11864                    compensation_registration_order: compensation.registration_order,
11865                });
11866            }
11867        }
11868        initiating_failure
11869    }
11870
11871    /// Return a successful forward value or compensate and preserve its failure.
11872    pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11873        match outcome {
11874            Ok(value) => Ok(value),
11875            Err(error) => Err(self.compensate(error).await),
11876        }
11877    }
11878}
11879
11880pub struct ActivityCall {
11881    ctx: WorkflowContext,
11882    activity_type: String,
11883    options: ActivityOptions,
11884    args: Option<Result<AvroValue>>,
11885    scheduled: bool,
11886    parallel_group_path: Vec<ParallelGroupMetadata>,
11887}
11888
11889impl ActivityCall {
11890    fn poll_avro_value(
11891        mut self: Pin<&mut Self>,
11892        _cx: &mut TaskContext<'_>,
11893    ) -> Poll<Result<AvroValue>> {
11894        let ctx = self.ctx.clone();
11895        let mut state = match ctx.state.lock() {
11896            Ok(state) => state,
11897            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11898        };
11899
11900        if self.scheduled {
11901            return Poll::Pending;
11902        }
11903
11904        let options = match self.options.validate() {
11905            Ok(options) => options,
11906            Err(error) => {
11907                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11908            }
11909        };
11910        let task_queue = options
11911            .task_queue
11912            .clone()
11913            .unwrap_or_else(|| state.task_queue.clone());
11914        let current_recorded_options = RecordedActivityOptions {
11915            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11916            // Rust schedules ordinary durable activities. The server records a
11917            // non-null mode only for a specialized execution primitive.
11918            execution_mode: RecordedSnapshotValue::Known(None),
11919            retry_policy: current_activity_retry_snapshot(&options),
11920        };
11921
11922        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11923            let sequence = recorded.sequence();
11924            match recorded {
11925                RecordedCommand::Activity {
11926                    activity_type,
11927                    options: recorded_options,
11928                    outcome,
11929                    parallel_group_path,
11930                    ..
11931                } => {
11932                    if let Err(error) = ensure_parallel_path_matches(
11933                        sequence,
11934                        parallel_group_path.as_deref(),
11935                        &self.parallel_group_path,
11936                    ) {
11937                        return Poll::Ready(Err(error));
11938                    }
11939                    if let Some(recorded_type) = activity_type {
11940                        if recorded_type != self.activity_type {
11941                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11942                                ReplayFailure::new(
11943                                    "recorded_command_detail_mismatch",
11944                                    Some(sequence),
11945                                    Some(format!("activity:{recorded_type}")),
11946                                    Some(format!("activity:{}", self.activity_type)),
11947                                    "recorded activity type differs from the current workflow command",
11948                                ),
11949                            )));
11950                        }
11951                    }
11952                    if let Some(recorded_options) = recorded_options {
11953                        if !recorded_options
11954                            .task_queue
11955                            .matches_current(&current_recorded_options.task_queue)
11956                        {
11957                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11958                                ReplayFailure::new(
11959                                    "activity_task_queue_mismatch",
11960                                    Some(sequence),
11961                                    Some(activity_options_description(&recorded_options)),
11962                                    Some(activity_options_description(&current_recorded_options)),
11963                                    "recorded activity task queue differs from the current workflow command",
11964                                ),
11965                            )));
11966                        }
11967                        if !recorded_options
11968                            .execution_mode
11969                            .matches_current(&current_recorded_options.execution_mode)
11970                        {
11971                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11972                                ReplayFailure::new(
11973                                    "activity_execution_mode_mismatch",
11974                                    Some(sequence),
11975                                    Some(activity_options_description(&recorded_options)),
11976                                    Some(activity_options_description(&current_recorded_options)),
11977                                    "recorded activity execution mode differs from the current workflow command",
11978                                ),
11979                            )));
11980                        }
11981                        if !recorded_options
11982                            .retry_policy
11983                            .matches_current(&current_recorded_options.retry_policy)
11984                        {
11985                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11986                                ReplayFailure::new(
11987                                    "activity_retry_policy_mismatch",
11988                                    Some(sequence),
11989                                    Some(activity_options_description(&recorded_options)),
11990                                    Some(activity_options_description(&current_recorded_options)),
11991                                    "recorded activity retry policy differs from the current workflow command",
11992                                ),
11993                            )));
11994                        }
11995                    }
11996                    state.command_cursor += 1;
11997                    if let Some(outcome) = outcome {
11998                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11999                    }
12000                    state.matched_recorded_pending = true;
12001                    self.scheduled = true;
12002                    return Poll::Pending;
12003                }
12004                other => {
12005                    return Poll::Ready(Err(command_mismatch(
12006                        &other,
12007                        format!("activity:{}", self.activity_type),
12008                    )));
12009                }
12010            }
12011        }
12012
12013        if !self.scheduled {
12014            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12015                Ok(args) => args,
12016                Err(error) => return Poll::Ready(Err(error)),
12017            };
12018            let arguments = normalize_avro_arguments(args);
12019            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
12020                Ok(envelope) => envelope,
12021                Err(error) => return Poll::Ready(Err(error)),
12022            };
12023
12024            let mut command = serde_json::Map::from_iter([
12025                ("type".to_string(), json!("schedule_activity")),
12026                (
12027                    "activity_type".to_string(),
12028                    json!(self.activity_type.clone()),
12029                ),
12030                ("queue".to_string(), json!(task_queue)),
12031                ("arguments".to_string(), envelope),
12032            ]);
12033            for (field, value) in [
12034                ("start_to_close_timeout", options.start_to_close_timeout),
12035                (
12036                    "schedule_to_start_timeout",
12037                    options.schedule_to_start_timeout,
12038                ),
12039                (
12040                    "schedule_to_close_timeout",
12041                    options.schedule_to_close_timeout,
12042                ),
12043                ("heartbeat_timeout", options.heartbeat_timeout),
12044            ] {
12045                if let Some(value) = value {
12046                    command.insert(field.to_string(), json!(value));
12047                }
12048            }
12049            if let Some(retry_policy) = options.retry_policy {
12050                command.insert("retry_policy".to_string(), retry_policy);
12051            }
12052            apply_parallel_group_path(&mut command, &self.parallel_group_path);
12053            state.commands.push(Value::Object(command));
12054            self.scheduled = true;
12055        }
12056
12057        Poll::Pending
12058    }
12059}
12060
12061impl Future for ActivityCall {
12062    type Output = Result<Value>;
12063
12064    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12065        match self.poll_avro_value(cx) {
12066            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
12067            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12068            Poll::Pending => Poll::Pending,
12069        }
12070    }
12071}
12072
12073/// Future returned by [`WorkflowContext::sleep`].
12074pub struct TimerCall {
12075    ctx: WorkflowContext,
12076    delay_seconds: Option<u64>,
12077    scheduled: bool,
12078    matched_pending: bool,
12079    parallel_group_path: Vec<ParallelGroupMetadata>,
12080}
12081
12082impl Future for TimerCall {
12083    type Output = Result<()>;
12084
12085    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12086        if self.matched_pending {
12087            return Poll::Pending;
12088        }
12089
12090        let ctx = self.ctx.clone();
12091        let Some(requested_delay) = self.delay_seconds else {
12092            return Poll::Ready(Err(Error::TimerDurationOverflow));
12093        };
12094        let mut state = match ctx.state.lock() {
12095            Ok(state) => state,
12096            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12097        };
12098
12099        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12100            match recorded {
12101                RecordedCommand::Timer {
12102                    sequence,
12103                    delay_seconds,
12104                    fired,
12105                    parallel_group_path,
12106                    ..
12107                } => {
12108                    if let Err(error) = ensure_parallel_path_matches(
12109                        sequence,
12110                        parallel_group_path.as_deref(),
12111                        &self.parallel_group_path,
12112                    ) {
12113                        return Poll::Ready(Err(error));
12114                    }
12115                    if delay_seconds != requested_delay {
12116                        return Poll::Ready(Err(Error::NonDeterministicReplay(
12117                            ReplayFailure::new(
12118                                "timer_delay_mismatch",
12119                                Some(sequence),
12120                                Some(format!("timer:{delay_seconds}s")),
12121                                Some(format!("timer:{requested_delay}s")),
12122                                "recorded timer delay differs from the current workflow command",
12123                            ),
12124                        )));
12125                    }
12126                    state.command_cursor += 1;
12127                    if fired {
12128                        return Poll::Ready(Ok(()));
12129                    }
12130                    state.matched_recorded_pending = true;
12131                    self.scheduled = true;
12132                    self.matched_pending = true;
12133                    return Poll::Pending;
12134                }
12135                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
12136            }
12137        }
12138
12139        if !self.scheduled {
12140            let mut command = serde_json::Map::from_iter([
12141                ("type".to_string(), json!("start_timer")),
12142                ("delay_seconds".to_string(), json!(requested_delay)),
12143            ]);
12144            apply_parallel_group_path(&mut command, &self.parallel_group_path);
12145            state.commands.push(Value::Object(command));
12146            self.scheduled = true;
12147        }
12148
12149        Poll::Pending
12150    }
12151}
12152
12153/// Future returned by [`WorkflowContext::wait_condition`].
12154pub struct ConditionWaitCall {
12155    ctx: WorkflowContext,
12156    options: ConditionWaitOptions,
12157    predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
12158    occurrence_id: Option<String>,
12159    opened_wait: bool,
12160    parallel_group_path: Vec<ParallelGroupMetadata>,
12161}
12162
12163impl Future for ConditionWaitCall {
12164    type Output = Result<ConditionWaitResult>;
12165
12166    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12167        if self.opened_wait {
12168            return Poll::Pending;
12169        }
12170
12171        let options = match self.options.validate() {
12172            Ok(options) => options,
12173            Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
12174        };
12175        let ctx = self.ctx.clone();
12176        let occurrence_id = match self.occurrence_id.as_ref() {
12177            Some(occurrence_id) => occurrence_id.clone(),
12178            None => {
12179                let mut state = match ctx.state.lock() {
12180                    Ok(state) => state,
12181                    Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12182                };
12183                let ordinal = state.condition_wait_occurrence_counter;
12184                state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
12185                    Some(next) => next,
12186                    None => {
12187                        return Poll::Ready(Err(Error::WorkerLoop(
12188                            "condition wait occurrence counter overflowed".to_string(),
12189                        )))
12190                    }
12191                };
12192                let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
12193                drop(state);
12194                self.occurrence_id = Some(occurrence_id.clone());
12195                occurrence_id
12196            }
12197        };
12198
12199        let recorded_result = {
12200            let mut state = match ctx.state.lock() {
12201                Ok(state) => state,
12202                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12203            };
12204            let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
12205                drop(state);
12206                return self.poll_new_condition(options);
12207            };
12208            if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
12209                return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
12210            }
12211
12212            let mut cursor = state.command_cursor;
12213            let mut result = None;
12214            loop {
12215                let Some(RecordedCommand::ConditionWait {
12216                    sequence,
12217                    occurrence_id: recorded_occurrence_id,
12218                    condition_key,
12219                    predicate_identity,
12220                    timeout_seconds,
12221                    result: recorded_result,
12222                    parallel_group_path,
12223                    ..
12224                }) = state.recorded_commands.get(cursor)
12225                else {
12226                    break;
12227                };
12228
12229                if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
12230                    break;
12231                }
12232                if let Err(error) = ensure_parallel_path_matches(
12233                    *sequence,
12234                    parallel_group_path.as_deref(),
12235                    &self.parallel_group_path,
12236                ) {
12237                    return Poll::Ready(Err(error));
12238                }
12239                if let Err(error) = validate_recorded_condition_wait(
12240                    *sequence,
12241                    recorded_occurrence_id,
12242                    condition_key.as_deref(),
12243                    predicate_identity,
12244                    *timeout_seconds,
12245                    &occurrence_id,
12246                    &options,
12247                ) {
12248                    return Poll::Ready(Err(error));
12249                }
12250                if result == Some(ConditionWaitResult::TimedOut) {
12251                    return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
12252                        "condition_wait_reopened_after_timeout",
12253                        Some(*sequence),
12254                        Some("timed-out condition is terminal".to_string()),
12255                        Some("another physical wait-open".to_string()),
12256                        "condition history reopened one logical wait after its durable timeout",
12257                    ))));
12258                }
12259                result = *recorded_result;
12260                cursor += 1;
12261            }
12262            state.command_cursor = cursor;
12263            result
12264        };
12265
12266        if let Some(result) = recorded_result {
12267            return Poll::Ready(Ok(result));
12268        }
12269
12270        self.poll_open_condition(options)
12271    }
12272}
12273
12274impl ConditionWaitCall {
12275    fn poll_new_condition(
12276        self: Pin<&mut Self>,
12277        options: ValidatedConditionWaitOptions,
12278    ) -> Poll<Result<ConditionWaitResult>> {
12279        self.poll_open_condition(options)
12280    }
12281
12282    fn poll_open_condition(
12283        mut self: Pin<&mut Self>,
12284        options: ValidatedConditionWaitOptions,
12285    ) -> Poll<Result<ConditionWaitResult>> {
12286        let selection_member = self
12287            .parallel_group_path
12288            .first()
12289            .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
12290        match (self.predicate)() {
12291            Ok(true) if !selection_member => {
12292                return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
12293            }
12294            Ok(_) => {}
12295            Err(error) => return Poll::Ready(Err(error)),
12296        }
12297        if options.timeout_seconds == Some(0) && !selection_member {
12298            return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
12299        }
12300
12301        let ctx = self.ctx.clone();
12302        let mut state = match ctx.state.lock() {
12303            Ok(state) => state,
12304            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12305        };
12306        let mut command = serde_json::Map::from_iter([
12307            ("type".to_string(), json!("open_condition_wait")),
12308            (
12309                "condition_wait_occurrence_id".to_string(),
12310                json!(self.occurrence_id.as_deref().unwrap_or_default()),
12311            ),
12312            ("condition_key".to_string(), json!(options.condition_key)),
12313            (
12314                "condition_definition_fingerprint".to_string(),
12315                json!(options.predicate_identity),
12316            ),
12317        ]);
12318        if let Some(timeout_seconds) = options.timeout_seconds {
12319            command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
12320        }
12321        apply_parallel_group_path(&mut command, &self.parallel_group_path);
12322        state.commands.push(Value::Object(command));
12323        drop(state);
12324        self.opened_wait = true;
12325        Poll::Pending
12326    }
12327}
12328
12329fn validate_recorded_condition_wait(
12330    sequence: u64,
12331    recorded_occurrence_id: &str,
12332    recorded_key: Option<&str>,
12333    recorded_predicate_identity: &str,
12334    recorded_timeout_seconds: Option<u64>,
12335    current_occurrence_id: &str,
12336    current: &ValidatedConditionWaitOptions,
12337) -> Result<()> {
12338    if recorded_occurrence_id != current_occurrence_id {
12339        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12340            "condition_wait_occurrence_mismatch",
12341            Some(sequence),
12342            Some(recorded_occurrence_id.to_string()),
12343            Some(current_occurrence_id.to_string()),
12344            "recorded condition occurrence differs from the current authored wait position",
12345        )));
12346    }
12347    if recorded_key != Some(current.condition_key.as_str()) {
12348        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12349            "condition_wait_key_mismatch",
12350            Some(sequence),
12351            recorded_key.map(str::to_string),
12352            Some(current.condition_key.clone()),
12353            "recorded condition identity differs from the current workflow wait",
12354        )));
12355    }
12356    if recorded_predicate_identity != current.predicate_identity {
12357        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12358            "condition_wait_predicate_mismatch",
12359            Some(sequence),
12360            Some(recorded_predicate_identity.to_string()),
12361            Some(current.predicate_identity.clone()),
12362            "recorded condition predicate behavior differs from current workflow code",
12363        )));
12364    }
12365    if recorded_timeout_seconds != current.timeout_seconds {
12366        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12367            "condition_wait_timeout_mismatch",
12368            Some(sequence),
12369            recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12370            current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12371            "recorded condition timeout differs from the current workflow wait",
12372        )));
12373    }
12374    Ok(())
12375}
12376
12377/// Future returned by [`WorkflowContext::start_child_workflow`].
12378pub struct ChildWorkflowCall {
12379    ctx: WorkflowContext,
12380    workflow_type: String,
12381    options: ChildWorkflowOptions,
12382    args: Option<Result<AvroValue>>,
12383    scheduled: bool,
12384    matched_pending: bool,
12385    parallel_group_path: Vec<ParallelGroupMetadata>,
12386}
12387
12388impl ChildWorkflowCall {
12389    fn poll_avro_value(
12390        mut self: Pin<&mut Self>,
12391        _cx: &mut TaskContext<'_>,
12392    ) -> Poll<Result<ChildWorkflowAvroResult>> {
12393        if self.matched_pending {
12394            return Poll::Pending;
12395        }
12396
12397        let ctx = self.ctx.clone();
12398        let mut state = match ctx.state.lock() {
12399            Ok(state) => state,
12400            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12401        };
12402
12403        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12404            let sequence = recorded.sequence();
12405            match recorded {
12406                RecordedCommand::ChildWorkflow {
12407                    workflow_type,
12408                    outcome,
12409                    parallel_group_path,
12410                    ..
12411                } => {
12412                    if let Err(error) = ensure_parallel_path_matches(
12413                        sequence,
12414                        parallel_group_path.as_deref(),
12415                        &self.parallel_group_path,
12416                    ) {
12417                        return Poll::Ready(Err(error));
12418                    }
12419                    if let Some(recorded_type) = workflow_type {
12420                        if recorded_type != self.workflow_type {
12421                            return Poll::Ready(Err(Error::NonDeterministicReplay(
12422                                ReplayFailure::new(
12423                                    "recorded_command_detail_mismatch",
12424                                    Some(sequence),
12425                                    Some(format!("child workflow:{recorded_type}")),
12426                                    Some(format!("child workflow:{}", self.workflow_type)),
12427                                    "recorded child workflow type differs from the current workflow command",
12428                                ),
12429                            )));
12430                        }
12431                    }
12432                    state.command_cursor += 1;
12433                    if let Some(outcome) = outcome {
12434                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12435                    }
12436                    state.matched_recorded_pending = true;
12437                    self.scheduled = true;
12438                    self.matched_pending = true;
12439                    return Poll::Pending;
12440                }
12441                other => {
12442                    return Poll::Ready(Err(command_mismatch(
12443                        &other,
12444                        format!("child workflow:{}", self.workflow_type),
12445                    )));
12446                }
12447            }
12448        }
12449
12450        if !self.scheduled {
12451            if self.options.task_queue.trim().is_empty() {
12452                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12453                    "task_queue must not be empty".to_string(),
12454                )));
12455            }
12456            for (name, value) in [
12457                (
12458                    "execution_timeout_seconds",
12459                    self.options.execution_timeout_seconds,
12460                ),
12461                ("run_timeout_seconds", self.options.run_timeout_seconds),
12462            ] {
12463                if value == Some(0) {
12464                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12465                        "{name} must be at least 1"
12466                    ))));
12467                }
12468            }
12469
12470            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12471                Ok(args) => args,
12472                Err(error) => return Poll::Ready(Err(error)),
12473            };
12474            let arguments = match encode_typed_envelope(
12475                &normalize_avro_arguments(args),
12476                &state.payload_codec,
12477            ) {
12478                Ok(arguments) => arguments,
12479                Err(error) => return Poll::Ready(Err(error)),
12480            };
12481            let mut command = json!({
12482                "type": "start_child_workflow",
12483                "workflow_type": self.workflow_type,
12484                "queue": self.options.task_queue,
12485                "parent_close_policy": self.options.parent_close_policy.as_str(),
12486                "arguments": arguments,
12487            });
12488            let object = command
12489                .as_object_mut()
12490                .expect("child workflow command is always an object");
12491            if let Some(policy) = &self.options.retry_policy {
12492                let mut retry_policy = serde_json::Map::new();
12493                if let Some(max_attempts) = policy.max_attempts {
12494                    if max_attempts == 0 {
12495                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12496                            "retry_policy.max_attempts must be at least 1".to_string(),
12497                        )));
12498                    }
12499                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12500                }
12501                if !policy.backoff_seconds.is_empty() {
12502                    retry_policy
12503                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12504                }
12505                if !policy.non_retryable_error_types.is_empty() {
12506                    retry_policy.insert(
12507                        "non_retryable_error_types".to_string(),
12508                        json!(policy.non_retryable_error_types),
12509                    );
12510                }
12511                if retry_policy.is_empty() {
12512                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12513                        "retry_policy must configure at least one field".to_string(),
12514                    )));
12515                }
12516                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12517            }
12518            if let Some(seconds) = self.options.execution_timeout_seconds {
12519                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12520            }
12521            if let Some(seconds) = self.options.run_timeout_seconds {
12522                object.insert("run_timeout_seconds".to_string(), json!(seconds));
12523            }
12524            apply_parallel_group_path(object, &self.parallel_group_path);
12525            state.commands.push(command);
12526            self.scheduled = true;
12527        }
12528
12529        Poll::Pending
12530    }
12531}
12532
12533impl Future for ChildWorkflowCall {
12534    type Output = Result<ChildWorkflowResult>;
12535
12536    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12537        match self.poll_avro_value(cx) {
12538            Poll::Ready(Ok(result)) => match result.result.into_json() {
12539                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12540                    parent: result.parent,
12541                    child: result.child,
12542                    child_workflow_type: result.child_workflow_type,
12543                    result: projected,
12544                })),
12545                Err(error) => Poll::Ready(Err(error)),
12546            },
12547            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12548            Poll::Pending => Poll::Pending,
12549        }
12550    }
12551}
12552
12553fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12554    Error::NonDeterministicReplay(ReplayFailure::new(
12555        "recorded_command_mismatch",
12556        Some(recorded.sequence()),
12557        Some(recorded.shape().to_string()),
12558        Some(actual.into()),
12559        "current workflow command does not match the recorded durable command sequence",
12560    ))
12561}
12562
12563pub struct SignalCall {
12564    ctx: WorkflowContext,
12565    signal_name: String,
12566    runtime_reserved_allowed: bool,
12567    opened_wait: bool,
12568    matched_pending: bool,
12569    parallel_group_path: Vec<ParallelGroupMetadata>,
12570}
12571
12572impl SignalCall {
12573    fn poll_avro_value(
12574        mut self: Pin<&mut Self>,
12575        _cx: &mut TaskContext<'_>,
12576    ) -> Poll<Result<Vec<AvroValue>>> {
12577        if self.matched_pending {
12578            return Poll::Pending;
12579        }
12580        if !self.runtime_reserved_allowed {
12581            if let Err(error) = validate_user_signal_name(&self.signal_name) {
12582                return Poll::Ready(Err(error));
12583            }
12584        }
12585
12586        let ctx = self.ctx.clone();
12587        let mut state = match ctx.state.lock() {
12588            Ok(state) => state,
12589            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12590        };
12591
12592        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12593            match recorded {
12594                RecordedCommand::SignalWait {
12595                    sequence,
12596                    signal_name,
12597                    value,
12598                    parallel_group_path,
12599                } => {
12600                    if let Err(error) = ensure_parallel_path_matches(
12601                        sequence,
12602                        parallel_group_path.as_deref(),
12603                        &self.parallel_group_path,
12604                    ) {
12605                        return Poll::Ready(Err(error));
12606                    }
12607                    if signal_name != self.signal_name {
12608                        return Poll::Ready(Err(Error::NonDeterministicReplay(
12609                            ReplayFailure::new(
12610                                "recorded_command_detail_mismatch",
12611                                Some(sequence),
12612                                Some(format!("signal wait:{signal_name}")),
12613                                Some(format!("signal wait:{}", self.signal_name)),
12614                                "recorded signal name differs from the current workflow command",
12615                            ),
12616                        )));
12617                    }
12618
12619                    state.command_cursor += 1;
12620                    if let Some(value) = value {
12621                        return Poll::Ready(Ok(value));
12622                    }
12623                    if state
12624                        .resume_signal
12625                        .as_ref()
12626                        .is_some_and(|signal| signal.signal_name == self.signal_name)
12627                    {
12628                        let signal = state
12629                            .resume_signal
12630                            .take()
12631                            .expect("matching resume signal is present");
12632                        return Poll::Ready(Ok(signal.arguments));
12633                    }
12634
12635                    state.matched_recorded_pending = true;
12636                    self.opened_wait = true;
12637                    self.matched_pending = true;
12638                    return Poll::Pending;
12639                }
12640                other => {
12641                    return Poll::Ready(Err(command_mismatch(
12642                        &other,
12643                        format!("signal wait:{}", self.signal_name),
12644                    )));
12645                }
12646            }
12647        }
12648
12649        if state
12650            .resume_signal
12651            .as_ref()
12652            .is_some_and(|signal| signal.signal_name == self.signal_name)
12653        {
12654            let signal = state
12655                .resume_signal
12656                .take()
12657                .expect("matching resume signal is present");
12658            return Poll::Ready(Ok(signal.arguments));
12659        }
12660
12661        if !self.opened_wait {
12662            let mut command = serde_json::Map::from_iter([
12663                ("type".to_string(), json!("open_signal_wait")),
12664                ("signal_name".to_string(), json!(self.signal_name)),
12665            ]);
12666            apply_parallel_group_path(&mut command, &self.parallel_group_path);
12667            state.commands.push(Value::Object(command));
12668            self.opened_wait = true;
12669        }
12670
12671        Poll::Pending
12672    }
12673}
12674
12675impl Future for SignalCall {
12676    type Output = Result<Vec<Value>>;
12677
12678    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12679        match self.poll_avro_value(cx) {
12680            Poll::Ready(Ok(values)) => Poll::Ready(
12681                values
12682                    .into_iter()
12683                    .map(AvroValue::into_json)
12684                    .collect::<Result<Vec<_>>>(),
12685            ),
12686            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12687            Poll::Pending => Poll::Pending,
12688        }
12689    }
12690}
12691
12692#[derive(Clone, Debug)]
12693pub struct ActivityContext {
12694    client: Client,
12695    pub task_id: String,
12696    pub activity_attempt_id: String,
12697    pub lease_owner: String,
12698    pub activity_type: String,
12699    pub attempt_number: u64,
12700    pub task_queue: String,
12701    pub worker_id: String,
12702}
12703
12704impl ActivityContext {
12705    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12706        self.client
12707            .heartbeat_activity_task(
12708                &self.task_id,
12709                &self.activity_attempt_id,
12710                &self.lease_owner,
12711                details,
12712            )
12713            .await
12714    }
12715}
12716
12717fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12718    validate_payload_codec(codec)?;
12719    match value {
12720        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12721            value, codec,
12722        )?)),
12723        None => Ok(AvroValue::Array(Vec::new())),
12724    }
12725}
12726
12727fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12728    let Some(signal_name) = task
12729        .signal_name
12730        .as_deref()
12731        .filter(|value| !value.is_empty())
12732    else {
12733        return Ok(None);
12734    };
12735    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12736    let AvroValue::Array(arguments) = decoded else {
12737        unreachable!("normalize_avro_arguments always returns an array");
12738    };
12739
12740    Ok(Some(ResumeSignal {
12741        signal_name: signal_name.to_string(),
12742        arguments,
12743    }))
12744}
12745
12746fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12747    validate_payload_codec(&task.payload_codec)?;
12748    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12749    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12750    for event in &task.history_events {
12751        validate_history_event_payloads(event, &task.payload_codec)?;
12752    }
12753    Ok(())
12754}
12755
12756fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12757    validate_payload_codec(&task.payload_codec)?;
12758    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12759}
12760
12761fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12762    validate_payload_codec(&task.payload_codec)?;
12763    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12764    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12765    for event in &task.history_events {
12766        validate_history_event_payloads(event, &task.payload_codec)?;
12767    }
12768
12769    let Some(export) = task.history_export.as_ref() else {
12770        return Ok(());
12771    };
12772    let export_codec = match export.get("payloads") {
12773        Some(payloads) => declared_payload_codec(payloads, "codec")?,
12774        None => None,
12775    }
12776    .unwrap_or(&task.payload_codec);
12777    validate_payload_codec(export_codec)?;
12778
12779    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12780        for event in events {
12781            let event_type = event
12782                .get("event_type")
12783                .or_else(|| event.get("type"))
12784                .and_then(Value::as_str)
12785                .unwrap_or_default();
12786            if let Some(payload) = event.get("payload") {
12787                validate_history_payloads(event_type, payload, export_codec)?;
12788            }
12789        }
12790    }
12791    for signal in export
12792        .get("signals")
12793        .and_then(Value::as_array)
12794        .into_iter()
12795        .flatten()
12796    {
12797        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12798        validate_payload_codec(codec)?;
12799        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12800    }
12801    for activity in export
12802        .get("activities")
12803        .and_then(Value::as_array)
12804        .into_iter()
12805        .flatten()
12806    {
12807        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12808        validate_payload_codec(codec)?;
12809        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12810        validate_optional_inbound_payload(activity.get("result"), codec)?;
12811    }
12812    Ok(())
12813}
12814
12815fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12816    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12817}
12818
12819fn validate_history_payloads(
12820    event_type: &str,
12821    payload: &Value,
12822    fallback_codec: &str,
12823) -> Result<()> {
12824    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12825    validate_payload_codec(codec)?;
12826    for field in history_payload_fields(event_type) {
12827        validate_optional_inbound_payload(payload.get(*field), codec)?;
12828    }
12829    Ok(())
12830}
12831
12832const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12833
12834fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12835    match event_type {
12836        "ActivityCompleted" => &["result"],
12837        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12838        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12839        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12840        "ChildRunCompleted" => &["result", "output"],
12841        "WorkflowCompleted" => &["output"],
12842        "ServiceCallStarted"
12843        | "ServiceCallCompleted"
12844        | "ServiceCallFailed"
12845        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12846        _ => &[],
12847    }
12848}
12849
12850fn signal_history_payload(payload: &Value) -> Option<&Value> {
12851    SIGNAL_HISTORY_PAYLOAD_FIELDS
12852        .iter()
12853        .find_map(|field| payload.get(*field))
12854}
12855
12856fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12857    match value.get(field) {
12858        None => Ok(None),
12859        Some(Value::String(codec)) => Ok(Some(codec)),
12860        Some(_) => Err(invalid_payload_envelope()),
12861    }
12862}
12863
12864fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12865    validate_payload_codec(codec)?;
12866    if let Some(value) = value.filter(|value| !value.is_null()) {
12867        decode_wire_avro_value(value, codec)?;
12868    }
12869    Ok(())
12870}
12871
12872fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12873    let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12874        invalid_recorded_history(
12875            "parallel_group_metadata_invalid",
12876            sequence,
12877            "non-empty parallel_group_id",
12878            &payload.to_string(),
12879            "parallel-group history is missing its stable identity",
12880        )
12881    })?;
12882    let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12883        invalid_recorded_history(
12884            "parallel_group_metadata_invalid",
12885            sequence,
12886            "activity, child, timer, signal, condition, or mixed group kind",
12887            &payload.to_string(),
12888            "parallel-group history is missing its group kind",
12889        )
12890    })?;
12891    if !matches!(
12892        kind.as_str(),
12893        "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12894    ) {
12895        return Err(invalid_recorded_history(
12896            "parallel_group_metadata_invalid",
12897            sequence,
12898            "activity, child, timer, signal, condition, or mixed group kind",
12899            &kind,
12900            "parallel-group history contains an unsupported group kind",
12901        ));
12902    }
12903    let base_sequence = payload
12904        .get("parallel_group_base_sequence")
12905        .and_then(value_as_u64)
12906        .filter(|value| *value > 0)
12907        .ok_or_else(|| {
12908            invalid_recorded_history(
12909                "parallel_group_metadata_invalid",
12910                sequence,
12911                "positive parallel_group_base_sequence",
12912                &payload.to_string(),
12913                "parallel-group history contains an invalid base sequence",
12914            )
12915        })?;
12916    let size = payload
12917        .get("parallel_group_size")
12918        .and_then(value_as_u64)
12919        .and_then(|value| usize::try_from(value).ok())
12920        .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12921        .ok_or_else(|| {
12922            invalid_recorded_history(
12923                "parallel_group_metadata_invalid",
12924                sequence,
12925                "bounded positive parallel_group_size",
12926                &payload.to_string(),
12927                "parallel-group history contains an invalid group size",
12928            )
12929        })?;
12930    let index = payload
12931        .get("parallel_group_index")
12932        .and_then(value_as_u64)
12933        .and_then(|value| usize::try_from(value).ok())
12934        .filter(|value| *value < size)
12935        .ok_or_else(|| {
12936            invalid_recorded_history(
12937                "parallel_group_metadata_invalid",
12938                sequence,
12939                "parallel_group_index within group bounds",
12940                &payload.to_string(),
12941                "parallel-group history contains an invalid member index",
12942            )
12943        })?;
12944    if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12945        return Err(invalid_recorded_history(
12946            "parallel_group_metadata_invalid",
12947            sequence,
12948            "base sequence plus member index equals workflow sequence",
12949            &payload.to_string(),
12950            "parallel-group path does not preserve durable workflow position",
12951        ));
12952    }
12953    let mode = payload
12954        .get("parallel_group_mode")
12955        .and_then(Value::as_str)
12956        .unwrap_or("all");
12957    if !matches!(mode, "all" | "select") {
12958        return Err(invalid_recorded_history(
12959            "parallel_group_metadata_invalid",
12960            sequence,
12961            "parallel group mode all or select",
12962            mode,
12963            "parallel-group history contains an unsupported group mode",
12964        ));
12965    }
12966    let expected_id = if mode == "select" {
12967        format!("select-calls:{base_sequence}:{size}")
12968    } else {
12969        format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12970    };
12971    if group_id != expected_id {
12972        return Err(invalid_recorded_history(
12973            "parallel_group_metadata_invalid",
12974            sequence,
12975            &expected_id,
12976            &group_id,
12977            "parallel-group history contains an incompatible stable group ID",
12978        ));
12979    }
12980    let selection_member_key = if mode == "select" {
12981        Some(selection_key_from_value(
12982            payload.get("selection_member_key"),
12983            sequence,
12984        )?)
12985    } else {
12986        None
12987    };
12988    let selection_member_index = if mode == "select" {
12989        Some(required_parallel_usize(
12990            payload,
12991            "selection_member_index",
12992            sequence,
12993        )?)
12994    } else {
12995        None
12996    };
12997    let selection_member_base_sequence = if mode == "select" {
12998        Some(
12999            payload
13000                .get("selection_member_base_sequence")
13001                .and_then(value_as_u64)
13002                .filter(|value| *value >= base_sequence)
13003                .ok_or_else(|| {
13004                    invalid_recorded_history(
13005                        "parallel_group_metadata_invalid",
13006                        sequence,
13007                        "selection member base within its group",
13008                        &payload.to_string(),
13009                        "selection history contains an invalid member base sequence",
13010                    )
13011                })?,
13012        )
13013    } else {
13014        None
13015    };
13016    let selection_member_size = if mode == "select" {
13017        let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
13018        if member_size == 0 {
13019            return Err(invalid_recorded_history(
13020                "parallel_group_metadata_invalid",
13021                sequence,
13022                "positive selection member size",
13023                &payload.to_string(),
13024                "selection history contains an invalid member size",
13025            ));
13026        }
13027        Some(member_size)
13028    } else {
13029        None
13030    };
13031    let selection_member_kind = if mode == "select" {
13032        let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
13033            invalid_recorded_history(
13034                "parallel_group_metadata_invalid",
13035                sequence,
13036                "selection member operation kind",
13037                &payload.to_string(),
13038                "selection history is missing its authored member kind",
13039            )
13040        })?;
13041        if !matches!(
13042            kind.as_str(),
13043            "activity" | "child" | "timer" | "signal" | "condition" | "group"
13044        ) {
13045            return Err(invalid_recorded_history(
13046                "parallel_group_metadata_invalid",
13047                sequence,
13048                "activity, child, timer, signal, condition, or group selection member kind",
13049                &kind,
13050                "selection history contains an unsupported member kind",
13051            ));
13052        }
13053        Some(kind)
13054    } else {
13055        None
13056    };
13057    if let (Some(member_base), Some(member_size)) =
13058        (selection_member_base_sequence, selection_member_size)
13059    {
13060        let member_end = member_base
13061            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
13062            .ok_or_else(|| {
13063                invalid_recorded_history(
13064                    "parallel_group_metadata_invalid",
13065                    sequence,
13066                    "bounded selection member range",
13067                    &payload.to_string(),
13068                    "selection member range overflowed",
13069                )
13070            })?;
13071        let group_end = base_sequence
13072            .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
13073            .unwrap_or(u64::MAX);
13074        if sequence < member_base || sequence >= member_end || member_end > group_end {
13075            return Err(invalid_recorded_history(
13076                "parallel_group_metadata_invalid",
13077                sequence,
13078                "workflow sequence within one bounded selection member",
13079                &payload.to_string(),
13080                "selection member range does not contain its durable leaf",
13081            ));
13082        }
13083    }
13084    Ok(ParallelGroupMetadata {
13085        parallel_group_id: group_id,
13086        parallel_group_kind: kind,
13087        parallel_group_base_sequence: base_sequence,
13088        parallel_group_size: size,
13089        parallel_group_index: index,
13090        parallel_group_mode: (mode == "select").then(|| "select".to_string()),
13091        selection_member_key,
13092        selection_member_index,
13093        selection_member_base_sequence,
13094        selection_member_size,
13095        selection_member_kind,
13096    })
13097}
13098
13099fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
13100    payload
13101        .get(field)
13102        .and_then(value_as_u64)
13103        .and_then(|value| usize::try_from(value).ok())
13104        .ok_or_else(|| {
13105            invalid_recorded_history(
13106                "parallel_group_metadata_invalid",
13107                sequence,
13108                &format!("non-negative integer {field}"),
13109                &payload.to_string(),
13110                "selection history contains invalid member metadata",
13111            )
13112        })
13113}
13114
13115fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
13116    match value {
13117        Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
13118        Some(value) => value_as_u64(value)
13119            .and_then(|value| usize::try_from(value).ok())
13120            .map(SelectionKey::Index)
13121            .ok_or_else(|| {
13122                invalid_recorded_history(
13123                    "selection_member_key_invalid",
13124                    sequence,
13125                    "non-empty string or non-negative integer member key",
13126                    &value.to_string(),
13127                    "selection history contains an invalid member key",
13128                )
13129            }),
13130        None => Err(invalid_recorded_history(
13131            "selection_member_key_missing",
13132            sequence,
13133            "selection_member_key",
13134            "<missing>",
13135            "selection history is missing its stable member key",
13136        )),
13137    }
13138}
13139
13140fn recorded_parallel_group_path(
13141    events: &[&HistoryEvent],
13142    sequence: u64,
13143) -> Result<Option<Vec<ParallelGroupMetadata>>> {
13144    let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
13145    for event in events {
13146        let payload = &event.payload;
13147        let has_metadata = payload.get("parallel_group_path").is_some()
13148            || payload.get("parallel_group_id").is_some()
13149            || payload.get("parallel_group_kind").is_some()
13150            || payload.get("parallel_group_base_sequence").is_some()
13151            || payload.get("parallel_group_size").is_some()
13152            || payload.get("parallel_group_index").is_some()
13153            || payload.get("parallel_group_mode").is_some()
13154            || payload.get("selection_member_key").is_some();
13155        if !has_metadata {
13156            continue;
13157        }
13158
13159        let top_level = recorded_parallel_group_entry(payload, sequence)?;
13160        let path = match payload.get("parallel_group_path") {
13161            None => vec![top_level.clone()],
13162            Some(Value::Array(entries)) if !entries.is_empty() => entries
13163                .iter()
13164                .map(|entry| recorded_parallel_group_entry(entry, sequence))
13165                .collect::<Result<Vec<_>>>()?,
13166            Some(value) => {
13167                return Err(invalid_recorded_history(
13168                    "parallel_group_metadata_invalid",
13169                    sequence,
13170                    "non-empty parallel_group_path list",
13171                    &value.to_string(),
13172                    "parallel-group history contains an invalid group path",
13173                ));
13174            }
13175        };
13176        if path.last() != Some(&top_level) {
13177            return Err(invalid_recorded_history(
13178                "parallel_group_metadata_invalid",
13179                sequence,
13180                &serde_json::to_string(&path.last()).unwrap_or_default(),
13181                &serde_json::to_string(&top_level).unwrap_or_default(),
13182                "parallel-group top-level fields do not match the innermost path entry",
13183            ));
13184        }
13185        if recorded.as_ref().is_some_and(|existing| existing != &path) {
13186            return Err(invalid_recorded_history(
13187                "parallel_group_history_conflict",
13188                sequence,
13189                &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
13190                &serde_json::to_string(&path).unwrap_or_default(),
13191                "parallel-group metadata changed between scheduling and resolution history",
13192            ));
13193        }
13194        recorded = Some(path);
13195    }
13196    Ok(recorded)
13197}
13198
13199fn recorded_commands(
13200    events: &[HistoryEvent],
13201    fallback_codec: &str,
13202    parent: WorkflowIdentity,
13203) -> Result<Vec<RecordedCommand>> {
13204    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
13205    let mut last_new_sequence = None;
13206
13207    for event in events {
13208        let is_activity = matches!(
13209            event.event_type.as_str(),
13210            "ActivityScheduled"
13211                | "ActivityStarted"
13212                | "ActivityHeartbeatRecorded"
13213                | "ActivityRetryScheduled"
13214                | "ActivityCompleted"
13215                | "ActivityFailed"
13216                | "ActivityCancelled"
13217                | "ActivityTimedOut"
13218        );
13219        let is_workflow_timer = matches!(
13220            event.event_type.as_str(),
13221            "TimerScheduled" | "TimerCancelled" | "TimerFired"
13222        ) && !is_internal_timer_event(event);
13223        let is_child_workflow = matches!(
13224            event.event_type.as_str(),
13225            "ChildWorkflowScheduled"
13226                | "ChildRunCompleted"
13227                | "ChildRunFailed"
13228                | "ChildRunCancelled"
13229                | "ChildRunTerminated"
13230        );
13231        let is_signal_wait = is_recorded_signal_wait_event(event);
13232        let is_condition_wait = is_recorded_condition_wait_event(event);
13233        let is_search_attributes = event.event_type == "SearchAttributesUpserted";
13234        let is_side_effect = event.event_type == "SideEffectRecorded";
13235        let is_version_marker = event.event_type == "VersionMarkerRecorded";
13236        let is_memo = event.event_type == "MemoUpserted";
13237        if !is_activity
13238            && !is_workflow_timer
13239            && !is_child_workflow
13240            && !is_signal_wait
13241            && !is_condition_wait
13242            && !is_search_attributes
13243            && !is_side_effect
13244            && !is_version_marker
13245            && !is_memo
13246        {
13247            continue;
13248        }
13249
13250        let sequence = durable_event_sequence(event).ok_or_else(|| {
13251            Error::NonDeterministicReplay(ReplayFailure::new(
13252                "durable_command_sequence_missing",
13253                None,
13254                Some("positive workflow sequence".to_string()),
13255                Some(event.event_type.clone()),
13256                "durable command history event has no workflow sequence",
13257            ))
13258        })?;
13259        if sequence == 0 {
13260            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
13261                "durable_command_sequence_invalid",
13262                Some(sequence),
13263                Some("positive workflow sequence".to_string()),
13264                Some(sequence.to_string()),
13265                "durable command history uses an invalid workflow sequence",
13266            )));
13267        }
13268        if !events_by_sequence.contains_key(&sequence) {
13269            if let Some(previous) = last_new_sequence {
13270                if sequence < previous {
13271                    return Err(invalid_recorded_history(
13272                        "durable_command_sequence_mismatch",
13273                        sequence,
13274                        &format!("workflow sequence greater than {previous}"),
13275                        &sequence.to_string(),
13276                        "durable commands are not strictly ordered by their recorded workflow sequence",
13277                    ));
13278                }
13279            }
13280            last_new_sequence = Some(sequence);
13281        }
13282        events_by_sequence.entry(sequence).or_default().push(event);
13283    }
13284
13285    let commands: Vec<RecordedCommand> = events_by_sequence
13286        .into_iter()
13287        .map(|(sequence, sequence_events)| {
13288            let activity_events: Vec<_> = sequence_events
13289                .iter()
13290                .copied()
13291                .filter(|event| event.event_type.starts_with("Activity"))
13292                .collect();
13293            let timer_events: Vec<_> = sequence_events
13294                .iter()
13295                .copied()
13296                .filter(|event| event.event_type.starts_with("Timer"))
13297                .collect();
13298            let child_events: Vec<_> = sequence_events
13299                .iter()
13300                .copied()
13301                .filter(|event| {
13302                    event.event_type == "ChildWorkflowScheduled"
13303                        || event.event_type.starts_with("ChildRun")
13304                })
13305                .collect();
13306            let signal_wait_events: Vec<_> = sequence_events
13307                .iter()
13308                .copied()
13309                .filter(|event| is_recorded_signal_wait_event(event))
13310                .collect();
13311            let condition_wait_events: Vec<_> = sequence_events
13312                .iter()
13313                .copied()
13314                .filter(|event| is_recorded_condition_wait_event(event))
13315                .collect();
13316            let search_attribute_events: Vec<_> = sequence_events
13317                .iter()
13318                .copied()
13319                .filter(|event| event.event_type == "SearchAttributesUpserted")
13320                .collect();
13321            let side_effect_events: Vec<_> = sequence_events
13322                .iter()
13323                .copied()
13324                .filter(|event| event.event_type == "SideEffectRecorded")
13325                .collect();
13326            let version_marker_events: Vec<_> = sequence_events
13327                .iter()
13328                .copied()
13329                .filter(|event| event.event_type == "VersionMarkerRecorded")
13330                .collect();
13331            let memo_events: Vec<_> = sequence_events
13332                .iter()
13333                .copied()
13334                .filter(|event| event.event_type == "MemoUpserted")
13335                .collect();
13336
13337            let command_kind_count = usize::from(!activity_events.is_empty())
13338                + usize::from(!timer_events.is_empty())
13339                + usize::from(!child_events.is_empty())
13340                + usize::from(!signal_wait_events.is_empty())
13341                + usize::from(!condition_wait_events.is_empty())
13342                + usize::from(!search_attribute_events.is_empty())
13343                + usize::from(!side_effect_events.is_empty())
13344                + usize::from(!version_marker_events.is_empty())
13345                + usize::from(!memo_events.is_empty());
13346            if command_kind_count > 1 {
13347                let actual = [
13348                    (!activity_events.is_empty()).then_some("activity"),
13349                    (!timer_events.is_empty()).then_some("timer"),
13350                    (!child_events.is_empty()).then_some("child workflow"),
13351                    (!signal_wait_events.is_empty()).then_some("signal wait"),
13352                    (!condition_wait_events.is_empty()).then_some("condition wait"),
13353                    (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13354                    (!side_effect_events.is_empty()).then_some("side effect"),
13355                    (!version_marker_events.is_empty()).then_some("version marker"),
13356                    (!memo_events.is_empty()).then_some("memo upsert"),
13357                ]
13358                .into_iter()
13359                .flatten()
13360                .collect::<Vec<_>>()
13361                .join(" and ");
13362                return Err(invalid_recorded_history(
13363                    "durable_command_sequence_collision",
13364                    sequence,
13365                    "one durable command kind",
13366                    &actual,
13367                    "one workflow sequence records more than one durable command kind",
13368                ));
13369            }
13370
13371            if !activity_events.is_empty() {
13372                let parallel_group_path =
13373                    recorded_parallel_group_path(&activity_events, sequence)?;
13374                let scheduled_count = activity_events
13375                    .iter()
13376                    .filter(|event| event.event_type == "ActivityScheduled")
13377                    .count();
13378                if scheduled_count > 1 {
13379                    return Err(invalid_recorded_history(
13380                        "duplicate_activity_schedule",
13381                        sequence,
13382                        "at most one ActivityScheduled event",
13383                        "multiple ActivityScheduled events",
13384                        "activity history schedules more than one command at one workflow sequence",
13385                    ));
13386                }
13387                let activity_type = activity_events.iter().find_map(|event| {
13388                    event
13389                        .payload
13390                        .get("activity_type")
13391                        .or_else(|| event.payload.get("activity_name"))
13392                        .and_then(Value::as_str)
13393                        .map(str::to_string)
13394                });
13395                if activity_events.iter().filter_map(|event| {
13396                    event
13397                        .payload
13398                        .get("activity_type")
13399                        .or_else(|| event.payload.get("activity_name"))
13400                        .and_then(Value::as_str)
13401                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13402                    return Err(invalid_recorded_history(
13403                        "activity_identity_mismatch",
13404                        sequence,
13405                        activity_type.as_deref().unwrap_or("one activity identity"),
13406                        "conflicting activity identities",
13407                        "activity lifecycle events at one workflow sequence disagree on identity",
13408                    ));
13409                }
13410                let terminal: Vec<_> = activity_events
13411                    .iter()
13412                    .copied()
13413                    .filter(|event| {
13414                        matches!(
13415                            event.event_type.as_str(),
13416                            "ActivityCompleted"
13417                                | "ActivityFailed"
13418                                | "ActivityCancelled"
13419                                | "ActivityTimedOut"
13420                        )
13421                    })
13422                    .collect();
13423                let duplicate_delivery = terminal.first().is_some_and(|first| {
13424                    terminal.iter().all(|event| {
13425                        event.event_type == first.event_type && event.payload == first.payload
13426                    })
13427                });
13428                if terminal.len() > 1 && !duplicate_delivery {
13429                    return Err(invalid_recorded_history(
13430                        "duplicate_activity_terminal_event",
13431                        sequence,
13432                        "at most one terminal activity event",
13433                        "multiple terminal activity events",
13434                        "activity history settles one command more than once",
13435                    ));
13436                }
13437                let outcome = terminal
13438                    .first()
13439                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13440                    .transpose()?;
13441                let options = activity_events
13442                    .iter()
13443                    .find(|event| event.event_type == "ActivityScheduled")
13444                    .and_then(|event| event.payload.get("activity"))
13445                    .and_then(Value::as_object)
13446                    .map(|activity| RecordedActivityOptions {
13447                        task_queue: recorded_optional_string(activity, "queue"),
13448                        execution_mode: recorded_optional_string(activity, "execution_mode"),
13449                        retry_policy: recorded_activity_retry_snapshot(
13450                            activity.get("retry_policy"),
13451                        ),
13452                    });
13453                return Ok(RecordedCommand::Activity {
13454                    sequence,
13455                    activity_type,
13456                    options,
13457                    outcome,
13458                    parallel_group_path,
13459                });
13460            }
13461
13462            if !child_events.is_empty() {
13463                let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13464                let scheduled: Vec<_> = child_events
13465                    .iter()
13466                    .copied()
13467                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
13468                    .collect();
13469                if scheduled.len() != 1 {
13470                    return Err(invalid_recorded_history(
13471                        "child_workflow_schedule_missing_or_duplicate",
13472                        sequence,
13473                        "one ChildWorkflowScheduled event",
13474                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13475                        "child workflow replay requires exactly one recorded schedule event",
13476                    ));
13477                }
13478                let workflow_type = child_events.iter().find_map(|event| {
13479                    event
13480                        .payload
13481                        .get("child_workflow_type")
13482                        .or_else(|| event.payload.get("workflow_type"))
13483                        .and_then(Value::as_str)
13484                        .filter(|value| !value.is_empty())
13485                        .map(str::to_string)
13486                });
13487                if child_events
13488                    .iter()
13489                    .filter_map(|event| {
13490                        event
13491                            .payload
13492                            .get("child_workflow_type")
13493                            .or_else(|| event.payload.get("workflow_type"))
13494                            .and_then(Value::as_str)
13495                    })
13496                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
13497                {
13498                    return Err(invalid_recorded_history(
13499                        "child_workflow_identity_mismatch",
13500                        sequence,
13501                        workflow_type
13502                            .as_deref()
13503                            .unwrap_or("one child workflow type"),
13504                        "conflicting child workflow types",
13505                        "child workflow lifecycle events at one sequence disagree on type",
13506                    ));
13507                }
13508                let mut outcomes = child_workflow_outcomes(
13509                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13510                    fallback_codec,
13511                    parent.clone(),
13512                )?;
13513                let terminal_events = child_events
13514                    .iter()
13515                    .copied()
13516                    .filter(|event| event.event_type.starts_with("ChildRun"))
13517                    .collect::<Vec<_>>();
13518                let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13519                    terminal_events.iter().all(|event| {
13520                        event.event_type == first.event_type && event.payload == first.payload
13521                    })
13522                });
13523                if outcomes.len() > 1 && !duplicate_delivery {
13524                    return Err(invalid_recorded_history(
13525                        "duplicate_child_workflow_terminal_event",
13526                        sequence,
13527                        "at most one terminal child event",
13528                        "multiple terminal child events",
13529                        "child workflow history settles one command more than once",
13530                    ));
13531                }
13532                return Ok(RecordedCommand::ChildWorkflow {
13533                    sequence,
13534                    workflow_type,
13535                    outcome: outcomes.pop(),
13536                    parallel_group_path,
13537                });
13538            }
13539
13540            if !signal_wait_events.is_empty() {
13541                let opened: Vec<_> = signal_wait_events
13542                    .iter()
13543                    .copied()
13544                    .filter(|event| event.event_type == "SignalWaitOpened")
13545                    .collect();
13546                if opened.len() != 1 {
13547                    return Err(invalid_recorded_history(
13548                        "signal_wait_open_missing_or_duplicate",
13549                        sequence,
13550                        "one SignalWaitOpened event",
13551                        &format!("{} SignalWaitOpened events", opened.len()),
13552                        "signal replay requires exactly one canonical wait-open event",
13553                    ));
13554                }
13555
13556                let applied: Vec<_> = signal_wait_events
13557                    .iter()
13558                    .copied()
13559                    .filter(|event| event.event_type == "SignalApplied")
13560                    .collect();
13561                if applied.len() > 1 {
13562                    return Err(invalid_recorded_history(
13563                        "duplicate_signal_wait_apply",
13564                        sequence,
13565                        "at most one SignalApplied event",
13566                        "multiple SignalApplied events",
13567                        "signal history applies one durable wait more than once",
13568                    ));
13569                }
13570
13571                let signal_names = signal_wait_events
13572                    .iter()
13573                    .map(|event| required_signal_wait_name(event, sequence))
13574                    .collect::<Result<Vec<_>>>()?;
13575                let signal_name = signal_names
13576                    .first()
13577                    .expect("signal wait events are not empty")
13578                    .clone();
13579                if signal_names.iter().any(|candidate| candidate != &signal_name) {
13580                    return Err(invalid_recorded_history(
13581                        "signal_wait_identity_mismatch",
13582                        sequence,
13583                        &signal_name,
13584                        "conflicting signal names",
13585                        "signal wait lifecycle events at one workflow sequence disagree on identity",
13586                    ));
13587                }
13588                let value = applied
13589                    .first()
13590                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
13591                    .transpose()?;
13592                return Ok(RecordedCommand::SignalWait {
13593                    sequence,
13594                    signal_name,
13595                    value,
13596                    parallel_group_path: recorded_parallel_group_path(
13597                        &signal_wait_events,
13598                        sequence,
13599                    )?,
13600                });
13601            }
13602
13603            if !condition_wait_events.is_empty() {
13604                return recorded_condition_wait(
13605                    sequence,
13606                    &condition_wait_events,
13607                    events,
13608                );
13609            }
13610
13611            if !search_attribute_events.is_empty() {
13612                if search_attribute_events.len() != 1 {
13613                    return Err(invalid_recorded_history(
13614                        "duplicate_search_attribute_update",
13615                        sequence,
13616                        "one SearchAttributesUpserted event",
13617                        &format!(
13618                            "{} SearchAttributesUpserted events",
13619                            search_attribute_events.len()
13620                        ),
13621                        "search-attribute history records one workflow command more than once",
13622                    ));
13623                }
13624                let payload = &search_attribute_events[0].payload;
13625                let attributes = payload
13626                    .get("attributes")
13627                    .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13628                    .cloned()
13629                    .ok_or_else(|| {
13630                        invalid_recorded_history(
13631                            "search_attribute_update_missing",
13632                            sequence,
13633                            "non-empty attributes object",
13634                            "missing or invalid attributes",
13635                            "search-attribute history is missing its recorded mutation",
13636                        )
13637                    })?;
13638                let attribute_types =
13639                    recorded_search_attribute_types(payload, &attributes, sequence)?;
13640                return Ok(RecordedCommand::SearchAttributes {
13641                    sequence,
13642                    attributes,
13643                    attribute_types,
13644                });
13645            }
13646
13647            if !side_effect_events.is_empty() {
13648                if side_effect_events.len() != 1 {
13649                    return Err(invalid_recorded_history(
13650                        "duplicate_side_effect_record",
13651                        sequence,
13652                        "one SideEffectRecorded event",
13653                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
13654                        "side-effect history records one workflow command more than once",
13655                    ));
13656                }
13657                let event = side_effect_events[0];
13658                let result = event.payload.get("result").ok_or_else(|| {
13659                    invalid_recorded_history(
13660                        "side_effect_result_missing",
13661                        sequence,
13662                        "recorded result payload",
13663                        "missing result",
13664                        "side-effect history is missing its recorded value",
13665                    )
13666                })?;
13667                let has_published_envelope = result.as_str().is_some()
13668                    || result.as_object().is_some_and(|envelope| {
13669                        envelope.get("codec").and_then(Value::as_str).is_some()
13670                            && envelope.get("blob").and_then(Value::as_str).is_some()
13671                    });
13672                if !has_published_envelope {
13673                    return Err(invalid_recorded_history(
13674                        "side_effect_payload_malformed",
13675                        sequence,
13676                        "payload blob or {codec, blob} envelope",
13677                        &result.to_string(),
13678                        "side-effect history result does not use a published payload envelope",
13679                    ));
13680                }
13681                let codec = event
13682                    .payload
13683                    .get("payload_codec")
13684                    .and_then(Value::as_str)
13685                    .unwrap_or(fallback_codec);
13686                let value = decode_wire_avro_value(result, codec).map_err(|error| {
13687                    if error.to_string().contains("unsupported_payload_codec") {
13688                        return error;
13689                    }
13690
13691                    invalid_recorded_history(
13692                        "side_effect_payload_incompatible",
13693                        sequence,
13694                        &format!("valid {codec} payload envelope"),
13695                        &error.to_string(),
13696                        "side-effect history payload cannot be decoded with its recorded codec",
13697                    )
13698                })?;
13699                return Ok(RecordedCommand::SideEffect { sequence, value });
13700            }
13701
13702            if !version_marker_events.is_empty() {
13703                if version_marker_events.len() != 1 {
13704                    return Err(invalid_recorded_history(
13705                        "duplicate_version_marker_record",
13706                        sequence,
13707                        "one VersionMarkerRecorded event",
13708                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13709                        "version-marker history records one workflow command more than once",
13710                    ));
13711                }
13712                let payload = &version_marker_events[0].payload;
13713                let change_id = payload
13714                    .get("change_id")
13715                    .and_then(Value::as_str)
13716                    .filter(|value| !value.is_empty())
13717                    .map(str::to_string)
13718                    .ok_or_else(|| {
13719                        invalid_recorded_history(
13720                            "version_marker_field_missing",
13721                            sequence,
13722                            "non-empty change_id",
13723                            "missing or invalid change_id",
13724                            "version-marker history is missing its stable change ID",
13725                        )
13726                    })?;
13727                let version = required_version_i32(payload, "version", sequence)?;
13728                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13729                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13730                if min_supported > max_supported || version < min_supported || version > max_supported {
13731                    return Err(invalid_recorded_history(
13732                        "version_marker_history_range_invalid",
13733                        sequence,
13734                        "min_supported <= version <= max_supported",
13735                        &format!("{min_supported} <= {version} <= {max_supported}"),
13736                        "recorded version marker contains an internally incompatible range",
13737                    ));
13738                }
13739                return Ok(RecordedCommand::VersionMarker {
13740                    sequence,
13741                    change_id,
13742                    version,
13743                });
13744            }
13745
13746            if !memo_events.is_empty() {
13747                if memo_events.len() != 1 {
13748                    return Err(invalid_recorded_history(
13749                        "duplicate_memo_upsert_record",
13750                        sequence,
13751                        "one MemoUpserted event",
13752                        &format!("{} MemoUpserted events", memo_events.len()),
13753                        "memo history records one workflow update more than once",
13754                    ));
13755                }
13756                let payload = &memo_events[0].payload;
13757                let entries = payload.get("entries").cloned().ok_or_else(|| {
13758                    invalid_recorded_history(
13759                        "memo_entries_missing",
13760                        sequence,
13761                        "memo entries object",
13762                        "missing entries",
13763                        "MemoUpserted history is missing replay identity entries",
13764                    )
13765                })?;
13766                let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13767                    invalid_recorded_history(
13768                        "memo_entries_invalid",
13769                        sequence,
13770                        "valid canonical memo entries",
13771                        &error.to_string(),
13772                        "MemoUpserted history contains invalid replay identity entries",
13773                    )
13774                })?;
13775                let merged = payload.get("merged").cloned().ok_or_else(|| {
13776                    invalid_recorded_history(
13777                        "memo_merged_projection_missing",
13778                        sequence,
13779                        "merged memo projection",
13780                        "missing merged",
13781                        "MemoUpserted history is missing its merged projection",
13782                    )
13783                })?;
13784                decode_memo_history_map(&merged, false).map_err(|error| {
13785                    invalid_recorded_history(
13786                        "memo_merged_projection_invalid",
13787                        sequence,
13788                        "valid merged memo projection",
13789                        &error.to_string(),
13790                        "MemoUpserted history contains an invalid merged projection",
13791                    )
13792                })?;
13793
13794                return Ok(RecordedCommand::Memo { sequence, entries });
13795            }
13796            let scheduled: Vec<_> = timer_events
13797                .iter()
13798                .copied()
13799                .filter(|event| event.event_type == "TimerScheduled")
13800                .collect();
13801            let fired: Vec<_> = timer_events
13802                .iter()
13803                .copied()
13804                .filter(|event| event.event_type == "TimerFired")
13805                .collect();
13806            if scheduled.len() != 1 {
13807                return Err(invalid_recorded_history(
13808                    "timer_schedule_missing_or_duplicate",
13809                    sequence,
13810                    "one TimerScheduled event",
13811                    &format!("{} TimerScheduled events", scheduled.len()),
13812                    "timer replay requires exactly one recorded schedule event",
13813                ));
13814            }
13815            if fired.len() > 1 {
13816                return Err(invalid_recorded_history(
13817                    "duplicate_timer_fire",
13818                    sequence,
13819                    "at most one TimerFired event",
13820                    "multiple TimerFired events",
13821                    "timer history contains more than one fire event for a workflow sequence",
13822                ));
13823            }
13824
13825            let scheduled = scheduled[0];
13826            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13827            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13828            if let Some(fired) = fired.first() {
13829                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13830                if fired_timer_id != timer_id {
13831                    return Err(invalid_recorded_history(
13832                        "timer_identity_mismatch",
13833                        sequence,
13834                        &timer_id,
13835                        &fired_timer_id,
13836                        "TimerFired does not correspond to the recorded TimerScheduled event",
13837                    ));
13838                }
13839                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13840                if fired_delay != delay_seconds {
13841                    return Err(invalid_recorded_history(
13842                        "timer_history_delay_mismatch",
13843                        sequence,
13844                        &delay_seconds.to_string(),
13845                        &fired_delay.to_string(),
13846                        "TimerScheduled and TimerFired record different delays",
13847                    ));
13848                }
13849            }
13850
13851            Ok(RecordedCommand::Timer {
13852                sequence,
13853                delay_seconds,
13854                fired: !fired.is_empty(),
13855                parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13856            })
13857        })
13858        .collect::<Result<_>>()?;
13859
13860    let mut marker_sequences = HashMap::new();
13861    for command in &commands {
13862        if let RecordedCommand::VersionMarker {
13863            sequence,
13864            change_id,
13865            ..
13866        } = command
13867        {
13868            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13869                return Err(invalid_recorded_history(
13870                    "duplicate_version_marker",
13871                    *sequence,
13872                    &format!("one marker for change ID {change_id:?}"),
13873                    &format!("markers at sequences {first_sequence} and {sequence}"),
13874                    "workflow history contains duplicate markers for one stable change ID",
13875                ));
13876            }
13877        }
13878    }
13879
13880    Ok(commands)
13881}
13882
13883fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13884    payload
13885        .get(field)
13886        .and_then(Value::as_i64)
13887        .and_then(|value| i32::try_from(value).ok())
13888        .ok_or_else(|| {
13889            invalid_recorded_history(
13890                "version_marker_field_missing",
13891                sequence,
13892                &format!("integer {field}"),
13893                "missing or out-of-range integer",
13894                "version-marker history is missing a required integer field",
13895            )
13896        })
13897}
13898
13899fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13900    event
13901        .payload
13902        .get("sequence")
13903        .or_else(|| event.payload.get("workflow_sequence"))
13904        .or_else(|| event.raw.get("sequence"))
13905        .or_else(|| event.raw.get("workflow_sequence"))
13906        .and_then(value_as_u64)
13907}
13908
13909fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13910    matches!(
13911        event
13912            .payload
13913            .get("timer_kind")
13914            .or_else(|| event.raw.get("timer_kind"))
13915            .and_then(Value::as_str),
13916        Some("condition_timeout" | "signal_timeout")
13917    )
13918}
13919
13920fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13921    matches!(
13922        event.event_type.as_str(),
13923        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13924    )
13925}
13926
13927fn recorded_condition_wait(
13928    sequence: u64,
13929    condition_events: &[&HistoryEvent],
13930    all_events: &[HistoryEvent],
13931) -> Result<RecordedCommand> {
13932    let opened = condition_events
13933        .iter()
13934        .copied()
13935        .filter(|event| event.event_type == "ConditionWaitOpened")
13936        .collect::<Vec<_>>();
13937    if opened.len() != 1 {
13938        return Err(invalid_recorded_history(
13939            "condition_wait_open_missing_or_duplicate",
13940            sequence,
13941            "one ConditionWaitOpened event",
13942            &format!("{} ConditionWaitOpened events", opened.len()),
13943            "condition replay requires exactly one canonical wait-open event",
13944        ));
13945    }
13946    let terminal = condition_events
13947        .iter()
13948        .copied()
13949        .filter(|event| {
13950            matches!(
13951                event.event_type.as_str(),
13952                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13953            )
13954        })
13955        .collect::<Vec<_>>();
13956    if terminal.len() > 1 {
13957        return Err(invalid_recorded_history(
13958            "duplicate_condition_wait_terminal_event",
13959            sequence,
13960            "at most one condition terminal event",
13961            "multiple condition terminal events",
13962            "condition history settles one durable wait more than once",
13963        ));
13964    }
13965
13966    let opened = opened[0];
13967    let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13968    let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13969    for event in condition_events
13970        .iter()
13971        .copied()
13972        .filter(|event| !std::ptr::eq(*event, opened))
13973    {
13974        let event_wait_id = required_condition_wait_id(event, sequence)?;
13975        if event_wait_id != condition_wait_id {
13976            return Err(invalid_recorded_history(
13977                "condition_wait_id_mismatch",
13978                sequence,
13979                &condition_wait_id,
13980                &event_wait_id,
13981                "condition lifecycle events at one sequence disagree on wait identity",
13982            ));
13983        }
13984        let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13985        if event_occurrence_id != occurrence_id {
13986            return Err(invalid_recorded_history(
13987                "condition_wait_occurrence_history_mismatch",
13988                sequence,
13989                &occurrence_id,
13990                &event_occurrence_id,
13991                "condition lifecycle events at one sequence disagree on authored occurrence identity",
13992            ));
13993        }
13994    }
13995
13996    let condition_key = optional_non_empty_history_string(opened, "condition_key");
13997    let predicate_identity = opened
13998        .payload
13999        .get("condition_definition_fingerprint")
14000        .and_then(Value::as_str)
14001        .filter(|value| !value.is_empty())
14002        .map(str::to_string)
14003        .ok_or_else(|| {
14004            invalid_recorded_history(
14005                "condition_wait_predicate_fingerprint_missing",
14006                sequence,
14007                "non-empty condition_definition_fingerprint",
14008                &opened.event_type,
14009                "canonical condition history is missing its predicate identity",
14010            )
14011        })?;
14012    let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
14013    for event in condition_events
14014        .iter()
14015        .copied()
14016        .filter(|event| !std::ptr::eq(*event, opened))
14017    {
14018        for (field, opened_value) in [
14019            ("condition_key", condition_key.as_deref()),
14020            (
14021                "condition_definition_fingerprint",
14022                Some(predicate_identity.as_str()),
14023            ),
14024        ] {
14025            if let Some(value) = optional_non_empty_history_string(event, field) {
14026                if opened_value.is_some_and(|opened_value| opened_value != value) {
14027                    return Err(invalid_recorded_history(
14028                        "condition_wait_definition_history_mismatch",
14029                        sequence,
14030                        opened_value.unwrap_or_default(),
14031                        &value,
14032                        "condition lifecycle events disagree on the recorded definition",
14033                    ));
14034                }
14035            }
14036        }
14037        if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
14038            if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
14039                return Err(invalid_recorded_history(
14040                    "condition_wait_definition_history_mismatch",
14041                    sequence,
14042                    &format!("{}s", timeout_seconds.unwrap_or_default()),
14043                    &format!("{event_timeout}s"),
14044                    "condition lifecycle events disagree on the recorded timeout",
14045                ));
14046            }
14047        }
14048    }
14049
14050    let timeout_timer_events = all_events
14051        .iter()
14052        .filter(|event| {
14053            matches!(
14054                event.event_type.as_str(),
14055                "TimerScheduled" | "TimerCancelled" | "TimerFired"
14056            ) && event.payload.get("timer_kind").and_then(Value::as_str)
14057                == Some("condition_timeout")
14058                && event
14059                    .payload
14060                    .get("condition_wait_id")
14061                    .and_then(Value::as_str)
14062                    == Some(condition_wait_id.as_str())
14063        })
14064        .collect::<Vec<_>>();
14065    let scheduled = timeout_timer_events
14066        .iter()
14067        .copied()
14068        .filter(|event| event.event_type == "TimerScheduled")
14069        .collect::<Vec<_>>();
14070    let fired = timeout_timer_events
14071        .iter()
14072        .copied()
14073        .filter(|event| event.event_type == "TimerFired")
14074        .collect::<Vec<_>>();
14075    if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
14076        return Err(invalid_recorded_history(
14077            "condition_wait_timeout_history_invalid",
14078            sequence,
14079            "one timeout schedule and at most one fire",
14080            &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
14081            "condition timeout history has a missing or duplicate lifecycle event",
14082        ));
14083    }
14084    if let Some(scheduled) = scheduled.first() {
14085        let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
14086        let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
14087        if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
14088            return Err(invalid_recorded_history(
14089                "condition_wait_timeout_delay_mismatch",
14090                sequence,
14091                &format!("{}s", timeout_seconds.unwrap_or_default()),
14092                &format!("{delay_seconds}s"),
14093                "condition timeout timer differs from the wait definition",
14094            ));
14095        }
14096        if let Some(fired) = fired.first() {
14097            let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
14098            let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
14099            if fired_timer_id != timer_id || fired_delay != delay_seconds {
14100                return Err(invalid_recorded_history(
14101                    "condition_wait_timeout_identity_mismatch",
14102                    sequence,
14103                    &format!("{timer_id}:{delay_seconds}s"),
14104                    &format!("{fired_timer_id}:{fired_delay}s"),
14105                    "condition timeout fire does not match its durable schedule",
14106                ));
14107            }
14108        }
14109    }
14110
14111    let result = terminal.first().map(|event| {
14112        if event.event_type == "ConditionWaitTimedOut" {
14113            ConditionWaitResult::TimedOut
14114        } else {
14115            ConditionWaitResult::Satisfied
14116        }
14117    });
14118    let result = if !fired.is_empty() {
14119        if result == Some(ConditionWaitResult::Satisfied) {
14120            return Err(invalid_recorded_history(
14121                "condition_wait_terminal_conflict",
14122                sequence,
14123                "one satisfied or timed-out outcome",
14124                "satisfied event and fired timeout",
14125                "condition history records conflicting terminal outcomes",
14126            ));
14127        }
14128        Some(ConditionWaitResult::TimedOut)
14129    } else {
14130        result
14131    };
14132
14133    Ok(RecordedCommand::ConditionWait {
14134        sequence,
14135        occurrence_id,
14136        condition_key,
14137        predicate_identity,
14138        timeout_seconds,
14139        result,
14140        parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
14141    })
14142}
14143
14144fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
14145    event
14146        .payload
14147        .get("condition_wait_occurrence_id")
14148        .and_then(Value::as_str)
14149        .filter(|value| !value.is_empty())
14150        .map(str::to_string)
14151        .ok_or_else(|| {
14152            invalid_recorded_history(
14153                "condition_wait_occurrence_id_missing",
14154                sequence,
14155                "non-empty condition_wait_occurrence_id",
14156                &event.event_type,
14157                "condition history is missing authored occurrence identity",
14158            )
14159        })
14160}
14161
14162fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
14163    event
14164        .payload
14165        .get("condition_wait_id")
14166        .and_then(Value::as_str)
14167        .filter(|value| !value.is_empty())
14168        .map(str::to_string)
14169        .ok_or_else(|| {
14170            invalid_recorded_history(
14171                "condition_wait_id_missing",
14172                sequence,
14173                "non-empty condition_wait_id",
14174                &event.event_type,
14175                "canonical condition history is missing its durable wait identity",
14176            )
14177        })
14178}
14179
14180fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
14181    event
14182        .payload
14183        .get(field)
14184        .and_then(Value::as_str)
14185        .filter(|value| !value.is_empty())
14186        .map(str::to_string)
14187}
14188
14189fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
14190    match event.payload.get(field) {
14191        None | Some(Value::Null) => Ok(None),
14192        Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
14193            invalid_recorded_history(
14194                "condition_wait_definition_invalid",
14195                sequence,
14196                &format!("non-negative integer {field}"),
14197                &value.to_string(),
14198                "condition history contains an invalid numeric definition field",
14199            )
14200        }),
14201    }
14202}
14203
14204fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
14205    event
14206        .payload
14207        .get("signal_name")
14208        .or_else(|| event.raw.get("signal_name"))
14209        .and_then(Value::as_str)
14210        .filter(|value| !value.is_empty())
14211        .map(str::to_string)
14212        .ok_or_else(|| {
14213            invalid_recorded_history(
14214                "signal_wait_name_missing",
14215                sequence,
14216                "non-empty signal_name",
14217                &event.event_type,
14218                "canonical signal-wait history is missing its signal identity",
14219            )
14220        })
14221}
14222
14223fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
14224    matches!(
14225        event.event_type.as_str(),
14226        "SignalWaitOpened" | "SignalApplied"
14227    )
14228}
14229
14230fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
14231    event
14232        .payload
14233        .get(field)
14234        .and_then(Value::as_str)
14235        .filter(|value| !value.is_empty())
14236        .map(str::to_string)
14237        .ok_or_else(|| {
14238            invalid_recorded_history(
14239                "timer_history_field_missing",
14240                sequence,
14241                field,
14242                &event.event_type,
14243                "timer history is missing a required identity field",
14244            )
14245        })
14246}
14247
14248fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
14249    event
14250        .payload
14251        .get(field)
14252        .and_then(value_as_u64)
14253        .ok_or_else(|| {
14254            invalid_recorded_history(
14255                "timer_history_field_missing",
14256                sequence,
14257                field,
14258                &event.event_type,
14259                "timer history is missing a required numeric field",
14260            )
14261        })
14262}
14263
14264fn recorded_search_attribute_types(
14265    payload: &Value,
14266    attributes: &Value,
14267    sequence: u64,
14268) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
14269    let Some(raw_types) = payload.get("attribute_types") else {
14270        // This is the explicit compatibility rule for histories recorded
14271        // before typed identity was persisted. Values still constrain replay;
14272        // the unknown type snapshot does not assert a typed match.
14273        return Ok(RecordedSnapshotValue::Unknown);
14274    };
14275    let Some(raw_types) = raw_types.as_object() else {
14276        return Err(invalid_recorded_history(
14277            "search_attribute_types_malformed",
14278            sequence,
14279            "canonical attribute type map",
14280            &raw_types.to_string(),
14281            "search-attribute history contains malformed type identity",
14282        ));
14283    };
14284    let attribute_keys = attributes
14285        .as_object()
14286        .expect("recorded search attributes were validated as an object");
14287    let mut types = BTreeMap::new();
14288    for (key, value) in raw_types {
14289        let Some(attribute_type) = value.as_str() else {
14290            return Err(invalid_recorded_history(
14291                "search_attribute_types_malformed",
14292                sequence,
14293                "canonical string type name",
14294                &value.to_string(),
14295                "search-attribute history contains a non-string type identity",
14296            ));
14297        };
14298        if !attribute_keys.contains_key(key)
14299            || !matches!(
14300                attribute_type,
14301                "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
14302            )
14303        {
14304            return Err(invalid_recorded_history(
14305                "search_attribute_types_malformed",
14306                sequence,
14307                "canonical types for keys present in attributes",
14308                &format!("{key}:{attribute_type}"),
14309                "search-attribute history contains unsupported or orphaned type identity",
14310            ));
14311        }
14312        types.insert(key.clone(), attribute_type.to_string());
14313    }
14314    Ok(RecordedSnapshotValue::Known(types))
14315}
14316
14317fn invalid_recorded_history(
14318    reason: &str,
14319    sequence: u64,
14320    expected: &str,
14321    actual: &str,
14322    message: &str,
14323) -> Error {
14324    Error::NonDeterministicReplay(ReplayFailure::new(
14325        reason,
14326        Some(sequence),
14327        Some(expected.to_string()),
14328        Some(actual.to_string()),
14329        message,
14330    ))
14331}
14332
14333type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
14334
14335fn activity_outcome(
14336    event: &HistoryEvent,
14337    fallback_codec: &str,
14338    recorded_activity_type: Option<String>,
14339) -> Result<ActivityOutcome> {
14340    if event.event_type == "ActivityCompleted" {
14341        let codec = event
14342            .payload
14343            .get("payload_codec")
14344            .and_then(Value::as_str)
14345            .unwrap_or(fallback_codec);
14346        return Ok(Ok(decode_wire_avro_value(
14347            event.payload.get("result").unwrap_or(&Value::Null),
14348            codec,
14349        )?));
14350    }
14351
14352    let payload = &event.payload;
14353    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14354        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14355        "ActivityCancelled" => (
14356            ActivityFailureKind::Cancelled,
14357            "cancelled",
14358            "activity was cancelled",
14359        ),
14360        "ActivityTimedOut" => (
14361            ActivityFailureKind::TimedOut,
14362            "timeout",
14363            "activity timed out",
14364        ),
14365        _ => unreachable!("activity_outcome is called only for terminal activity events"),
14366    };
14367    let exception = payload
14368        .get("exception")
14369        .filter(|value| !value.is_null())
14370        .cloned();
14371    let failure_category = payload_string(payload, "failure_category");
14372    let timeout_kind = payload_string(payload, "timeout_kind");
14373    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14374        ActivityFailureKind::Failed => failure_category
14375            .clone()
14376            .unwrap_or_else(|| fallback_reason.to_string()),
14377        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14378        ActivityFailureKind::TimedOut => timeout_kind
14379            .clone()
14380            .unwrap_or_else(|| fallback_reason.to_string()),
14381    });
14382    let message = payload_string(payload, "message")
14383        .or_else(|| {
14384            exception
14385                .as_ref()
14386                .and_then(|value| payload_string(value, "message"))
14387        })
14388        .unwrap_or_else(|| fallback_message.to_string());
14389
14390    Ok(Err(ActivityFailure {
14391        kind,
14392        reason,
14393        message,
14394        activity_execution_id: payload_string(payload, "activity_execution_id"),
14395        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14396        activity_type: payload_string(payload, "activity_type")
14397            .or_else(|| payload_string(payload, "activity_name"))
14398            .or(recorded_activity_type),
14399        activity_class: payload_string(payload, "activity_class"),
14400        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14401        failure_id: payload_string(payload, "failure_id"),
14402        failure_category,
14403        timeout_kind,
14404        non_retryable: payload
14405            .get("non_retryable")
14406            .and_then(Value::as_bool)
14407            .unwrap_or(false),
14408        exception_type: payload_string(payload, "exception_type").or_else(|| {
14409            exception
14410                .as_ref()
14411                .and_then(|value| payload_string(value, "type"))
14412        }),
14413        exception_class: payload_string(payload, "exception_class").or_else(|| {
14414            exception
14415                .as_ref()
14416                .and_then(|value| payload_string(value, "class"))
14417        }),
14418        code: payload
14419            .get("code")
14420            .filter(|value| !value.is_null())
14421            .cloned(),
14422        exception,
14423    }))
14424}
14425
14426type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14427
14428fn child_workflow_outcomes(
14429    events: &[HistoryEvent],
14430    fallback_codec: &str,
14431    parent: WorkflowIdentity,
14432) -> Result<Vec<ChildWorkflowOutcome>> {
14433    let mut outcomes = Vec::new();
14434
14435    for event in events {
14436        let kind = match event.event_type.as_str() {
14437            "ChildRunCompleted" => None,
14438            "ChildRunFailed" => Some((
14439                ChildWorkflowFailureKind::Failed,
14440                "child_workflow",
14441                "child workflow failed",
14442            )),
14443            "ChildRunCancelled" => Some((
14444                ChildWorkflowFailureKind::Cancelled,
14445                "cancelled",
14446                "child workflow was cancelled",
14447            )),
14448            "ChildRunTerminated" => Some((
14449                ChildWorkflowFailureKind::Terminated,
14450                "terminated",
14451                "child workflow was terminated",
14452            )),
14453            _ => continue,
14454        };
14455        let payload = &event.payload;
14456        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14457        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14458        let child_workflow_type = payload_string(payload, "child_workflow_type");
14459
14460        if let Some((kind, reason, fallback_message)) = kind {
14461            let exception = payload
14462                .get("exception")
14463                .filter(|value| !value.is_null())
14464                .cloned();
14465            let message = payload_string(payload, "message")
14466                .or_else(|| {
14467                    exception
14468                        .as_ref()
14469                        .and_then(|value| payload_string(value, "message"))
14470                })
14471                .unwrap_or_else(|| fallback_message.to_string());
14472            let exception_type = payload_string(payload, "exception_type").or_else(|| {
14473                exception
14474                    .as_ref()
14475                    .and_then(|value| payload_string(value, "type"))
14476            });
14477            let exception_class = payload_string(payload, "exception_class").or_else(|| {
14478                exception
14479                    .as_ref()
14480                    .and_then(|value| payload_string(value, "class"))
14481            });
14482            outcomes.push(Err(ChildWorkflowFailure {
14483                kind,
14484                reason: reason.to_string(),
14485                message,
14486                parent_workflow_id: parent.workflow_id.clone(),
14487                parent_workflow_run_id: parent.run_id.clone(),
14488                child_workflow_id,
14489                child_workflow_run_id,
14490                child_workflow_type,
14491                failure_id: payload_string(payload, "failure_id"),
14492                failure_category: payload_string(payload, "failure_category"),
14493                exception_type,
14494                exception_class,
14495                non_retryable: payload
14496                    .get("non_retryable")
14497                    .and_then(Value::as_bool)
14498                    .unwrap_or(false),
14499                code: payload
14500                    .get("code")
14501                    .filter(|value| !value.is_null())
14502                    .cloned(),
14503                exception,
14504            }));
14505            continue;
14506        }
14507
14508        let codec = payload
14509            .get("payload_codec")
14510            .and_then(Value::as_str)
14511            .unwrap_or(fallback_codec);
14512        let result = payload
14513            .get("result")
14514            .or_else(|| payload.get("output"))
14515            .unwrap_or(&Value::Null);
14516        outcomes.push(Ok(ChildWorkflowAvroResult {
14517            parent: parent.clone(),
14518            child: WorkflowIdentity {
14519                workflow_id: child_workflow_id,
14520                run_id: child_workflow_run_id,
14521            },
14522            child_workflow_type,
14523            result: decode_wire_avro_value(result, codec)?,
14524        }));
14525    }
14526
14527    Ok(outcomes)
14528}
14529
14530fn payload_string(payload: &Value, key: &str) -> Option<String> {
14531    payload
14532        .get(key)
14533        .and_then(Value::as_str)
14534        .filter(|value| !value.is_empty())
14535        .map(str::to_string)
14536}
14537
14538fn recorded_activity_failure_boundary(
14539    ctx: &WorkflowContext,
14540    error: &Error,
14541) -> Option<(u64, String)> {
14542    let Error::ActivityFailed(failure) = error else {
14543        return None;
14544    };
14545    if failure.kind != ActivityFailureKind::Failed {
14546        return None;
14547    }
14548    let activity_execution_id = failure
14549        .activity_execution_id
14550        .as_deref()
14551        .filter(|value| !value.is_empty())?;
14552    let state = ctx.state.lock().ok()?;
14553    let mut matches = state.recorded_commands.iter().filter_map(|command| {
14554        let RecordedCommand::Activity {
14555            sequence,
14556            outcome: Some(Err(recorded_failure)),
14557            ..
14558        } = command
14559        else {
14560            return None;
14561        };
14562        (*sequence > 0 && recorded_failure == failure)
14563            .then_some((*sequence, activity_execution_id.to_string()))
14564    });
14565    let boundary = matches.next()?;
14566    matches.next().is_none().then_some(boundary)
14567}
14568
14569fn workflow_failure_command(
14570    error: &Error,
14571    failed_activity_boundary: Option<(u64, String)>,
14572) -> Value {
14573    let (exception_type, exception_class, properties) = match error {
14574        Error::ActivityFailed(failure) => (
14575            match failure.kind {
14576                ActivityFailureKind::Failed => "ActivityFailed",
14577                ActivityFailureKind::Cancelled => "ActivityCancelled",
14578                ActivityFailureKind::TimedOut => "ActivityTimedOut",
14579            },
14580            "durable_workflow::ActivityFailure",
14581            json!({
14582                "reason": failure.reason,
14583                "activity_execution_id": failure.activity_execution_id,
14584                "activity_attempt_id": failure.activity_attempt_id,
14585                "activity_type": failure.activity_type,
14586                "activity_class": failure.activity_class,
14587                "attempt_number": failure.attempt_number,
14588                "failure_id": failure.failure_id,
14589                "failure_category": failure.failure_category,
14590                "timeout_kind": failure.timeout_kind,
14591                "activity_non_retryable": failure.non_retryable,
14592                "activity_exception_type": failure.exception_type,
14593                "activity_exception_class": failure.exception_class,
14594                "activity_code": failure.code,
14595                "activity_exception": failure.exception,
14596            }),
14597        ),
14598        Error::ChildWorkflowFailed(failure) => (
14599            match failure.kind {
14600                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14601                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14602                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14603            },
14604            "durable_workflow::ChildWorkflowFailure",
14605            json!({
14606                "reason": failure.reason,
14607                "parent_workflow_id": failure.parent_workflow_id,
14608                "parent_workflow_run_id": failure.parent_workflow_run_id,
14609                "child_workflow_id": failure.child_workflow_id,
14610                "child_workflow_run_id": failure.child_workflow_run_id,
14611                "child_workflow_type": failure.child_workflow_type,
14612                "failure_id": failure.failure_id,
14613                "failure_category": failure.failure_category,
14614                "child_exception_type": failure.exception_type,
14615                "child_exception_class": failure.exception_class,
14616                "child_non_retryable": failure.non_retryable,
14617                "child_code": failure.code,
14618                "child_exception": failure.exception,
14619            }),
14620        ),
14621        Error::ParallelFailed(failure) => (
14622            "ParallelFailed",
14623            "durable_workflow::ParallelFailure",
14624            json!({
14625                "parallel_group_id": failure.group_id,
14626                "parallel_member_path": failure.member_path,
14627                "parallel_group_path": failure.group_path,
14628                "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14629                "cause_type": workflow_error_type(&failure.cause),
14630                "cause_message": failure.cause.to_string(),
14631            }),
14632        ),
14633        Error::SagaCompensationFailed(failure) => (
14634            "SagaCompensationFailed",
14635            "durable_workflow::SagaCompensationFailure",
14636            json!({
14637                "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14638                "initiating_failure_message": failure.initiating_failure.to_string(),
14639                "compensation_activity_type": failure.compensation_activity_type,
14640                "compensation_registration_order": failure.compensation_registration_order,
14641                "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14642                "compensation_failure_message": failure.compensation_failure.to_string(),
14643            }),
14644        ),
14645        Error::WorkflowCancellationRequested(_) => (
14646            "WorkflowCancellationRequested",
14647            "durable_workflow::WorkflowCancellationRequested",
14648            json!({"reason": "cancelled"}),
14649        ),
14650        Error::NonDeterministicReplay(_) => (
14651            "NonDeterministicReplay",
14652            "durable_workflow::Error",
14653            Value::Null,
14654        ),
14655        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14656    };
14657    let non_retryable = match error {
14658        Error::ActivityFailed(failure) => failure.non_retryable,
14659        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14660        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14661        Error::SagaCompensationFailed(failure) => {
14662            workflow_error_non_retryable(&failure.compensation_failure)
14663        }
14664        Error::WorkflowCancellationRequested(_) => true,
14665        Error::NonDeterministicReplay(_) => true,
14666        _ => false,
14667    };
14668
14669    let mut command = json!({
14670        "type": "fail_workflow",
14671        "message": error.to_string(),
14672        "exception_type": exception_type,
14673        "exception_class": exception_class,
14674        "non_retryable": non_retryable,
14675        "exception": {
14676            "type": exception_type,
14677            "class": exception_class,
14678            "message": error.to_string(),
14679            "properties": properties,
14680        }
14681    });
14682
14683    if let Some((sequence, activity_execution_id)) = failed_activity_boundary {
14684        command["failed_step_sequence"] = json!(sequence);
14685        command["failed_activity_execution_id"] = json!(activity_execution_id);
14686    }
14687
14688    command
14689}
14690
14691fn workflow_error_type(error: &Error) -> &'static str {
14692    match error {
14693        Error::ActivityFailed(failure) => match failure.kind {
14694            ActivityFailureKind::Failed => "ActivityFailed",
14695            ActivityFailureKind::Cancelled => "ActivityCancelled",
14696            ActivityFailureKind::TimedOut => "ActivityTimedOut",
14697        },
14698        Error::ChildWorkflowFailed(failure) => match failure.kind {
14699            ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14700            ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14701            ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14702        },
14703        Error::ParallelFailed(_) => "ParallelFailed",
14704        Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14705        Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14706        Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14707        _ => "RustWorkflowError",
14708    }
14709}
14710
14711fn workflow_error_non_retryable(error: &Error) -> bool {
14712    match error {
14713        Error::ActivityFailed(failure) => failure.non_retryable,
14714        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14715        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14716        Error::SagaCompensationFailed(failure) => {
14717            workflow_error_non_retryable(&failure.compensation_failure)
14718        }
14719        Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14720        _ => false,
14721    }
14722}
14723
14724fn workflow_task_integrity_error(error: &Error) -> bool {
14725    matches!(
14726        error,
14727        Error::NonDeterministicReplay(_)
14728            | Error::Protocol(_)
14729            | Error::MissingWorkflowCommandIdentity
14730            | Error::WorkflowStatePoisoned
14731    )
14732}
14733
14734fn decode_signal_event_arguments(
14735    event: &HistoryEvent,
14736    fallback_codec: &str,
14737) -> Result<Vec<AvroValue>> {
14738    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14739    validate_payload_codec(codec)?;
14740    let raw = signal_history_payload(&event.payload);
14741    let decoded = match raw.filter(|value| !value.is_null()) {
14742        Some(value) => decode_wire_avro_value(value, codec)?,
14743        None => AvroValue::Array(Vec::new()),
14744    };
14745    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14746        unreachable!("normalize_avro_arguments always returns an array");
14747    };
14748    Ok(arguments)
14749}
14750
14751fn decode_update_event_arguments(
14752    event: &HistoryEvent,
14753    fallback_codec: &str,
14754) -> Result<Vec<AvroValue>> {
14755    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14756    validate_payload_codec(codec)?;
14757    let decoded = match event
14758        .payload
14759        .get("arguments")
14760        .filter(|value| !value.is_null())
14761    {
14762        Some(value) => decode_wire_avro_value(value, codec)?,
14763        None => AvroValue::Array(Vec::new()),
14764    };
14765    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14766        unreachable!("normalize_avro_arguments always returns an array");
14767    };
14768    Ok(arguments)
14769}
14770
14771fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14772    let Some(export_events) = task
14773        .history_export
14774        .as_ref()
14775        .and_then(|export| export.get("history_events"))
14776        .and_then(Value::as_array)
14777    else {
14778        return Ok(());
14779    };
14780
14781    if export_events.len() > task.history_events.len() {
14782        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14783    }
14784
14785    Ok(())
14786}
14787
14788fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14789    let Some(export) = task.history_export.as_ref() else {
14790        return Ok(());
14791    };
14792    let signals = export
14793        .get("signals")
14794        .and_then(Value::as_array)
14795        .cloned()
14796        .unwrap_or_default();
14797    let activities = export
14798        .get("activities")
14799        .and_then(Value::as_array)
14800        .cloned()
14801        .unwrap_or_default();
14802    let export_codec = export
14803        .get("payloads")
14804        .and_then(|payloads| payloads.get("codec"))
14805        .and_then(Value::as_str)
14806        .unwrap_or(&task.payload_codec)
14807        .to_string();
14808    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14809
14810    for event in &mut task.history_events {
14811        if event.event_type == "ActivityCompleted" {
14812            let sequence = event
14813                .payload
14814                .get("sequence")
14815                .or_else(|| event.payload.get("workflow_sequence"))
14816                .and_then(value_as_u64);
14817            let Some(activity) = sequence.and_then(|sequence| {
14818                activities.iter().find(|activity| {
14819                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14820                })
14821            }) else {
14822                continue;
14823            };
14824            let Some(payload) = event.payload.as_object_mut() else {
14825                continue;
14826            };
14827            if missing_payload(payload.get("result")) {
14828                if let Some(result) = activity
14829                    .get("result")
14830                    .filter(|value| !missing_payload(Some(value)))
14831                {
14832                    payload.insert("result".to_string(), result.clone());
14833                }
14834            }
14835            for field in ["payload_codec", "activity_type"] {
14836                if payload
14837                    .get(field)
14838                    .and_then(Value::as_str)
14839                    .unwrap_or_default()
14840                    .is_empty()
14841                {
14842                    if let Some(value) = activity.get(field) {
14843                        payload.insert(field.to_string(), value.clone());
14844                    }
14845                }
14846            }
14847            continue;
14848        }
14849
14850        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14851            continue;
14852        }
14853        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14854        let command_id = event
14855            .payload
14856            .get("workflow_command_id")
14857            .or_else(|| event.raw.get("workflow_command_id"))
14858            .and_then(Value::as_str);
14859        let signal_name = event
14860            .payload
14861            .get("signal_name")
14862            .and_then(Value::as_str)
14863            .unwrap_or_default()
14864            .to_string();
14865        let matched = signals
14866            .iter()
14867            .find(|signal| {
14868                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14869            })
14870            .or_else(|| {
14871                signals.iter().find(|signal| {
14872                    command_id.is_some()
14873                        && signal.get("command_id").and_then(Value::as_str) == command_id
14874                })
14875            })
14876            .or_else(|| {
14877                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14878                let signal = signals
14879                    .iter()
14880                    .filter(|signal| {
14881                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14882                    })
14883                    .nth(*offset);
14884                if signal.is_some() {
14885                    *offset += 1;
14886                }
14887                signal
14888            });
14889        let Some(signal) = matched else {
14890            continue;
14891        };
14892        let signal_codec = signal
14893            .get("payload_codec")
14894            .and_then(Value::as_str)
14895            .unwrap_or(&export_codec);
14896        let Some(payload) = event.payload.as_object_mut() else {
14897            continue;
14898        };
14899        if missing_payload(payload.get("arguments")) {
14900            if let Some(arguments) = signal
14901                .get("arguments")
14902                .filter(|value| !missing_payload(Some(value)))
14903            {
14904                let envelope = match arguments {
14905                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14906                    other => other.clone(),
14907                };
14908                payload.insert("arguments".to_string(), envelope);
14909            }
14910        }
14911        if payload
14912            .get("payload_codec")
14913            .and_then(Value::as_str)
14914            .unwrap_or_default()
14915            .is_empty()
14916        {
14917            payload.insert("payload_codec".to_string(), json!(signal_codec));
14918        }
14919    }
14920
14921    Ok(())
14922}
14923
14924fn missing_payload(value: Option<&Value>) -> bool {
14925    match value {
14926        None | Some(Value::Null) => true,
14927        Some(Value::String(value)) => value.is_empty(),
14928        Some(_) => false,
14929    }
14930}
14931
14932fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14933    let export_signals = task
14934        .history_export
14935        .as_ref()
14936        .and_then(|export| export.get("signals"))
14937        .and_then(Value::as_array)
14938        .cloned()
14939        .unwrap_or_default();
14940    let export_codec = task
14941        .history_export
14942        .as_ref()
14943        .and_then(|export| export.get("payloads"))
14944        .and_then(|payloads| payloads.get("codec"))
14945        .and_then(Value::as_str)
14946        .unwrap_or(&task.payload_codec);
14947    let mut name_offsets: HashMap<String, usize> = HashMap::new();
14948    let mut signals = Vec::new();
14949
14950    for event in &task.history_events {
14951        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14952            continue;
14953        }
14954
14955        let name = event
14956            .payload
14957            .get("signal_name")
14958            .and_then(Value::as_str)
14959            .unwrap_or_default();
14960        if name.is_empty() {
14961            continue;
14962        }
14963        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14964        let command_id = event
14965            .payload
14966            .get("workflow_command_id")
14967            .or_else(|| event.raw.get("workflow_command_id"))
14968            .and_then(Value::as_str);
14969        let matched_export = export_signals
14970            .iter()
14971            .find(|candidate| {
14972                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14973            })
14974            .or_else(|| {
14975                export_signals.iter().find(|candidate| {
14976                    command_id.is_some()
14977                        && candidate.get("command_id").and_then(Value::as_str) == command_id
14978                })
14979            })
14980            .or_else(|| {
14981                let offset = name_offsets.entry(name.to_string()).or_default();
14982                let candidate = export_signals
14983                    .iter()
14984                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14985                    .nth(*offset);
14986                if candidate.is_some() {
14987                    *offset += 1;
14988                }
14989                candidate
14990            });
14991        let codec = event
14992            .payload
14993            .get("payload_codec")
14994            .and_then(Value::as_str)
14995            .or_else(|| {
14996                matched_export
14997                    .and_then(|signal| signal.get("payload_codec"))
14998                    .and_then(Value::as_str)
14999            })
15000            .unwrap_or(export_codec);
15001        let raw_arguments = signal_history_payload(&event.payload)
15002            .filter(|value| !value.is_null())
15003            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
15004        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
15005        let workflow_sequence = event
15006            .payload
15007            .get("workflow_sequence")
15008            .and_then(value_as_u64)
15009            .or_else(|| {
15010                matched_export
15011                    .and_then(|signal| signal.get("workflow_sequence"))
15012                    .and_then(value_as_u64)
15013            });
15014
15015        signals.push(QuerySignal {
15016            id: signal_id.map(str::to_string).or_else(|| {
15017                matched_export
15018                    .and_then(|signal| signal.get("id"))
15019                    .and_then(Value::as_str)
15020                    .map(str::to_string)
15021            }),
15022            name: name.to_string(),
15023            arguments,
15024            avro_arguments,
15025            workflow_sequence,
15026        });
15027    }
15028
15029    if signals.is_empty() {
15030        for signal in export_signals {
15031            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
15032                continue;
15033            }
15034            let Some(name) = signal.get("name").and_then(Value::as_str) else {
15035                continue;
15036            };
15037            let codec = signal
15038                .get("payload_codec")
15039                .and_then(Value::as_str)
15040                .unwrap_or(export_codec);
15041            let (arguments, avro_arguments) =
15042                decode_query_signal_arguments(signal.get("arguments"), codec)?;
15043            signals.push(QuerySignal {
15044                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
15045                name: name.to_string(),
15046                arguments,
15047                avro_arguments,
15048                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
15049            });
15050        }
15051        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
15052    }
15053
15054    Ok(signals)
15055}
15056
15057fn decode_query_signal_arguments(
15058    raw: Option<&Value>,
15059    codec: &str,
15060) -> Result<(Vec<Value>, Vec<AvroValue>)> {
15061    validate_payload_codec(codec)?;
15062    let decoded = match raw.filter(|value| !value.is_null()) {
15063        Some(value) => decode_wire_avro_value(value, codec)?,
15064        None => AvroValue::Array(Vec::new()),
15065    };
15066    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
15067        unreachable!("normalize_avro_arguments always returns an array");
15068    };
15069    let arguments = avro_arguments
15070        .iter()
15071        .cloned()
15072        .map(AvroValue::into_json)
15073        .collect::<Result<Vec<_>>>()?;
15074    Ok((arguments, avro_arguments))
15075}
15076
15077fn value_as_u64(value: &Value) -> Option<u64> {
15078    value
15079        .as_u64()
15080        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
15081}
15082
15083#[cfg(test)]
15084mod tests {
15085    use super::*;
15086    mod runtime_payloads;
15087    mod runtime_uploads;
15088    use std::{
15089        fs,
15090        io::{Read, Write},
15091        net::{SocketAddr, TcpListener, TcpStream},
15092        process::Command as ProcessCommand,
15093        sync::atomic::AtomicUsize,
15094        thread,
15095    };
15096
15097    #[derive(Clone, Copy, Debug)]
15098    enum InvalidTaskPayloadCodec {
15099        Missing,
15100        Null,
15101        NonString,
15102    }
15103
15104    impl InvalidTaskPayloadCodec {
15105        fn label(self) -> &'static str {
15106            match self {
15107                Self::Missing => "missing",
15108                Self::Null => "null",
15109                Self::NonString => "non-string",
15110            }
15111        }
15112
15113        fn apply(self, task: &mut Value) {
15114            let task = task.as_object_mut().expect("task fixture object");
15115            match self {
15116                Self::Missing => {
15117                    task.remove("payload_codec");
15118                }
15119                Self::Null => {
15120                    task.insert("payload_codec".to_string(), Value::Null);
15121                }
15122                Self::NonString => {
15123                    task.insert("payload_codec".to_string(), json!(42));
15124                }
15125            }
15126        }
15127    }
15128
15129    fn fixture_envelope(value: Value) -> Value {
15130        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
15131    }
15132
15133    fn fixture_blob(value: Value) -> String {
15134        encode_payload(&value, DEFAULT_CODEC)
15135            .expect("encode Avro test fixture")
15136            .blob
15137    }
15138
15139    #[test]
15140    fn client_builder_rejects_the_sdk_owned_api_suffix() {
15141        for base_url in [
15142            "http://127.0.0.1:8080/api",
15143            "http://localhost:8080/api/",
15144            "https://runtime.example.test/namespaces/orders/api",
15145        ] {
15146            let error = Client::builder(base_url)
15147                .build()
15148                .expect_err("SDK-owned /api suffix must be rejected during build");
15149
15150            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
15151            assert!(
15152                error.to_string().contains("SDK appends /api automatically"),
15153                "the validation error must explain how to fix the endpoint"
15154            );
15155        }
15156    }
15157
15158    #[test]
15159    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
15160        for (base_url, expected) in [
15161            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
15162            (
15163                "http://localhost:8080/durable-workflow/",
15164                "http://localhost:8080/durable-workflow",
15165            ),
15166            (
15167                "https://runtime.example.test/namespaces/orders",
15168                "https://runtime.example.test/namespaces/orders",
15169            ),
15170            (
15171                "https://runtime.example.test/gateway/api/namespaces/orders",
15172                "https://runtime.example.test/gateway/api/namespaces/orders",
15173            ),
15174            (
15175                "https://api.example.test/runtime/orders/",
15176                "https://api.example.test/runtime/orders",
15177            ),
15178        ] {
15179            let client = Client::builder(base_url)
15180                .build()
15181                .expect("Server and Cloud runtime base URL must remain valid");
15182
15183            assert_eq!(client.base_url, expected);
15184        }
15185    }
15186
15187    #[test]
15188    fn workflow_completion_uses_the_additive_command_protocol_floor() {
15189        assert_eq!(
15190            workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
15191            WORKER_PROTOCOL_VERSION
15192        );
15193        assert_eq!(
15194            workflow_completion_protocol_version(&[json!({
15195                "type": "upsert_search_attributes",
15196                "attributes": {"OrderStatus": "waiting"},
15197            })]),
15198            SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
15199        );
15200        assert_eq!(
15201            workflow_completion_protocol_version(&[json!({
15202                "type": "upsert_search_attributes",
15203                "attributes": {"OrderStatus": "waiting"},
15204                "attribute_types": {"OrderStatus": "keyword"},
15205            })]),
15206            TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
15207        );
15208        assert_eq!(
15209            workflow_completion_protocol_version(&[
15210                json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
15211                json!({"type": "open_condition_wait", "condition_key": "ready"}),
15212            ]),
15213            MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
15214        );
15215        assert_eq!(
15216            workflow_completion_protocol_version(&[
15217                json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
15218                json!({"type": "open_condition_wait", "condition_key": "ready"}),
15219            ]),
15220            CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
15221        );
15222        assert_eq!(
15223            workflow_completion_protocol_version(&[json!({
15224                "type": "open_condition_wait",
15225                "condition_wait_occurrence_id": "rust:condition-wait:0",
15226                "condition_key": "ready",
15227            })]),
15228            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
15229        );
15230        assert_eq!(
15231            workflow_completion_protocol_version_with_message_streams(
15232                &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
15233                true,
15234            ),
15235            MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
15236        );
15237        assert_eq!(
15238            workflow_completion_protocol_version_with_message_streams(
15239                &[json!({
15240                    "type": "open_condition_wait",
15241                    "condition_wait_occurrence_id": "rust:condition-wait:0",
15242                    "condition_key": "ready",
15243                })],
15244                true,
15245            ),
15246            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
15247        );
15248    }
15249
15250    #[test]
15251    fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
15252        let manifest = portable_worker_affinity_capability_manifest();
15253
15254        for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
15255            assert_eq!(manifest[capability]["supported"], json!(false));
15256            assert_eq!(
15257                manifest[capability]["minimum_protocol_version"],
15258                json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
15259            );
15260            assert!(manifest[capability]["reason"]
15261                .as_str()
15262                .is_some_and(|reason| !reason.is_empty()));
15263        }
15264    }
15265
15266    fn typed_fidelity_probe() -> AvroValue {
15267        AvroValue::Map(BTreeMap::from([
15268            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
15269            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
15270            (
15271                "numeric".to_string(),
15272                AvroValue::Map(BTreeMap::from([
15273                    ("0".to_string(), AvroValue::String("zero".to_string())),
15274                    ("1".to_string(), AvroValue::String("one".to_string())),
15275                ])),
15276            ),
15277            (
15278                "nested".to_string(),
15279                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
15280                    "enabled".to_string(),
15281                    AvroValue::Boolean(true),
15282                )]))]),
15283            ),
15284            (
15285                "projection_collisions".to_string(),
15286                AvroValue::Array(projection_collision_probe()),
15287            ),
15288        ]))
15289    }
15290
15291    fn projection_collision_probe() -> Vec<AvroValue> {
15292        vec![
15293            AvroValue::Map(BTreeMap::from([
15294                ("$type".to_string(), AvroValue::String("bytes".to_string())),
15295                (
15296                    "base64".to_string(),
15297                    AvroValue::String("ordinary user text".to_string()),
15298                ),
15299            ])),
15300            AvroValue::Map(BTreeMap::from([
15301                ("$type".to_string(), AvroValue::String("map".to_string())),
15302                (
15303                    "entries".to_string(),
15304                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
15305                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
15306                        (
15307                            "value".to_string(),
15308                            AvroValue::String("user map".to_string()),
15309                        ),
15310                    ]))]),
15311                ),
15312            ])),
15313        ]
15314    }
15315
15316    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15317    struct TypedContract {
15318        nested: TypedNested,
15319        mode: TypedMode,
15320        optional: Option<String>,
15321        absent: Option<String>,
15322        items: Vec<i64>,
15323        labels: BTreeMap<String, String>,
15324        bytes: serde_bytes::ByteBuf,
15325        signed: i64,
15326        finite: f64,
15327    }
15328
15329    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15330    struct TypedNested {
15331        enabled: bool,
15332    }
15333
15334    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15335    enum TypedMode {
15336        Detailed { label: String },
15337    }
15338
15339    fn typed_contract() -> TypedContract {
15340        TypedContract {
15341            nested: TypedNested { enabled: true },
15342            mode: TypedMode::Detailed {
15343                label: "compiler-checked".to_string(),
15344            },
15345            optional: Some("present".to_string()),
15346            absent: None,
15347            items: vec![i64::MIN, 0, i64::MAX],
15348            labels: BTreeMap::from([
15349                ("language".to_string(), "rust".to_string()),
15350                ("wire".to_string(), "avro".to_string()),
15351            ]),
15352            bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
15353            signed: -9_223_372_036_854_775_000,
15354            finite: 12.5,
15355        }
15356    }
15357
15358    #[derive(Clone, Debug, Default, PartialEq)]
15359    struct ReplayCounterState {
15360        loaded: Option<String>,
15361        count: i64,
15362        finished: bool,
15363    }
15364
15365    fn replay_counter_worker() -> Worker {
15366        let client = Client::new("http://127.0.0.1:8080").expect("client");
15367        let mut worker = Worker::new(client, "rust-workers");
15368        worker.register_replayed_workflow(
15369            "replay-counter",
15370            ReplayCounterState::default,
15371            |ctx, _input, state| async move {
15372                let loaded = ctx.activity("load-counter", json!([])).await?;
15373                state.update(|current| {
15374                    current.loaded = loaded.as_str().map(str::to_string);
15375                })?;
15376                for _ in 0..2 {
15377                    let signal = ctx.wait_signal("increment").await?;
15378                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15379                    state.update(|current| current.count += amount)?;
15380                }
15381                state.update(|current| current.finished = true)?;
15382                state.read(|current| Ok(json!(current.count)))?
15383            },
15384        );
15385        worker.register_replayed_query::<ReplayCounterState, _, _>(
15386            "replay-counter",
15387            "current",
15388            |_ctx, state, _args| async move {
15389                Ok(json!({
15390                    "loaded": state.loaded,
15391                    "count": state.count,
15392                    "finished": state.finished,
15393                }))
15394            },
15395        );
15396        worker.register_replayed_query::<ReplayCounterState, _, _>(
15397            "replay-counter",
15398            "detached-mutation",
15399            |_ctx, state, _args| async move {
15400                let mut detached = (*state).clone();
15401                detached.count = 999;
15402                Ok(json!(detached.count))
15403            },
15404        );
15405        worker.register_replayed_query::<ReplayCounterState, _, _>(
15406            "replay-counter",
15407            "failed-mutation",
15408            |_ctx, state, _args| async move {
15409                let mut detached = (*state).clone();
15410                detached.count = 999;
15411                Err(Error::WorkerLoop("query refused".to_string()))
15412            },
15413        );
15414        worker
15415    }
15416
15417    fn replay_counter_query(
15418        query_name: &str,
15419        history_events: Value,
15420        run_status: &str,
15421    ) -> QueryTask {
15422        let arguments = fixture_envelope(json!([]));
15423        serde_json::from_value(json!({
15424            "query_task_id": format!("query-{query_name}"),
15425            "workflow_type": "replay-counter",
15426            "query_name": query_name,
15427            "payload_codec": DEFAULT_CODEC,
15428            "workflow_arguments": arguments.clone(),
15429            "query_arguments": arguments,
15430            "history_events": history_events,
15431            "run_status": run_status,
15432        }))
15433        .expect("query task")
15434    }
15435
15436    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15437        workflow_context_with_codec(history, DEFAULT_CODEC)
15438    }
15439
15440    fn workflow_context_with_codec(
15441        history: Vec<HistoryEvent>,
15442        payload_codec: &str,
15443    ) -> WorkflowContext {
15444        WorkflowContext {
15445            state: Arc::new(Mutex::new(
15446                WorkflowState::new_with_identity(
15447                    history,
15448                    None,
15449                    None,
15450                    "rust-workers".to_string(),
15451                    payload_codec.to_string(),
15452                    None,
15453                )
15454                .expect("valid workflow history"),
15455            )),
15456        }
15457    }
15458
15459    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15460        HistoryEvent {
15461            event_type: event_type.to_string(),
15462            payload,
15463            raw: HashMap::new(),
15464        }
15465    }
15466
15467    fn parallel_path_entry(
15468        kind: &str,
15469        base: u64,
15470        size: usize,
15471        index: usize,
15472    ) -> ParallelGroupMetadata {
15473        parallel_group_entry(base, size, index, kind)
15474    }
15475
15476    fn parallel_history_event(
15477        event_type: &str,
15478        sequence: u64,
15479        identity_field: &str,
15480        identity: &str,
15481        path: Vec<ParallelGroupMetadata>,
15482        result: Option<Value>,
15483    ) -> HistoryEvent {
15484        let mut payload = serde_json::Map::from_iter([
15485            ("sequence".to_string(), json!(sequence)),
15486            (identity_field.to_string(), json!(identity)),
15487        ]);
15488        let inner = path.last().expect("parallel history path");
15489        apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15490        payload.insert("parallel_group_path".to_string(), json!(path));
15491        if let Some(result) = result {
15492            let field = if event_type == "ChildRunCompleted" {
15493                "result"
15494            } else {
15495                "result"
15496            };
15497            payload.insert(field.to_string(), fixture_envelope(result));
15498            payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15499        }
15500        history_event(event_type, Value::Object(payload))
15501    }
15502
15503    fn nested_parallel_operations() -> Vec<ParallelOperation> {
15504        vec![
15505            ParallelOperation::activity("first", json!([])),
15506            ParallelOperation::group(vec![
15507                ParallelOperation::child_workflow(
15508                    "second",
15509                    ChildWorkflowOptions::new("child-workers"),
15510                    json!([]),
15511                ),
15512                ParallelOperation::activity("third", json!([])),
15513            ]),
15514        ]
15515    }
15516
15517    fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15518        let outer = [
15519            parallel_path_entry("mixed", 1, 3, 0),
15520            parallel_path_entry("mixed", 1, 3, 1),
15521            parallel_path_entry("mixed", 1, 3, 2),
15522        ];
15523        [
15524            vec![outer[0].clone()],
15525            vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15526            vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15527        ]
15528    }
15529
15530    #[test]
15531    fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15532        let ctx = workflow_context(Vec::new());
15533        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15534        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15535
15536        assert!(matches!(
15537            call.as_mut().poll(&mut task_context),
15538            Poll::Pending
15539        ));
15540        let commands = ctx.take_commands().expect("parallel commands");
15541        assert_eq!(
15542            commands
15543                .iter()
15544                .map(|command| command["type"].as_str().unwrap_or_default())
15545                .collect::<Vec<_>>(),
15546            [
15547                "schedule_activity",
15548                "start_child_workflow",
15549                "schedule_activity"
15550            ]
15551        );
15552        let paths = nested_parallel_paths();
15553        for (command, path) in commands.iter().zip(paths) {
15554            assert_eq!(command["parallel_group_path"], json!(path));
15555            assert_eq!(
15556                command["parallel_group_id"],
15557                json!(path.last().expect("inner group").parallel_group_id)
15558            );
15559        }
15560    }
15561
15562    fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15563        let paths = nested_parallel_paths();
15564        let third = parallel_history_event(
15565            "ActivityCompleted",
15566            3,
15567            "activity_type",
15568            "third",
15569            paths[2].clone(),
15570            Some(json!("three")),
15571        );
15572        vec![
15573            parallel_history_event(
15574                "ActivityCompleted",
15575                1,
15576                "activity_type",
15577                "first",
15578                paths[0].clone(),
15579                Some(json!("one")),
15580            ),
15581            parallel_history_event(
15582                "ChildWorkflowScheduled",
15583                2,
15584                "child_workflow_type",
15585                "second",
15586                paths[1].clone(),
15587                None,
15588            ),
15589            parallel_history_event(
15590                "ChildRunCompleted",
15591                2,
15592                "child_workflow_type",
15593                "second",
15594                paths[1].clone(),
15595                Some(json!("two")),
15596            ),
15597            third.clone(),
15598            third,
15599        ]
15600    }
15601
15602    #[test]
15603    fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15604        for _restart_or_completed_replay in 0..2 {
15605            let ctx = workflow_context(completed_nested_parallel_history());
15606            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15607            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15608            let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15609                panic!("completed nested parallel history must replay");
15610            };
15611            assert_eq!(
15612                results,
15613                vec![
15614                    ParallelResult::Activity(json!("one")),
15615                    ParallelResult::Group(vec![
15616                        ParallelResult::ChildWorkflow(ChildWorkflowResult {
15617                            parent: WorkflowIdentity {
15618                                workflow_id: None,
15619                                run_id: None,
15620                            },
15621                            child: WorkflowIdentity {
15622                                workflow_id: None,
15623                                run_id: None,
15624                            },
15625                            child_workflow_type: Some("second".to_string()),
15626                            result: json!("two"),
15627                        }),
15628                        ParallelResult::Activity(json!("three")),
15629                    ]),
15630                ]
15631            );
15632            assert!(ctx.take_commands().expect("commands").is_empty());
15633            ctx.ensure_history_consumed().expect("history consumed");
15634        }
15635    }
15636
15637    #[test]
15638    fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15639        let paths = nested_parallel_paths();
15640        let history = vec![
15641            parallel_history_event(
15642                "ActivityCompleted",
15643                1,
15644                "activity_type",
15645                "first",
15646                paths[0].clone(),
15647                Some(json!("one")),
15648            ),
15649            parallel_history_event(
15650                "ChildWorkflowScheduled",
15651                2,
15652                "child_workflow_type",
15653                "second",
15654                paths[1].clone(),
15655                None,
15656            ),
15657            parallel_history_event(
15658                "ChildRunFailed",
15659                2,
15660                "child_workflow_type",
15661                "second",
15662                paths[1].clone(),
15663                None,
15664            ),
15665            parallel_history_event(
15666                "ActivityCompleted",
15667                3,
15668                "activity_type",
15669                "third",
15670                paths[2].clone(),
15671                Some(json!("late")),
15672            ),
15673        ];
15674        let ctx = workflow_context(history);
15675        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15676        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15677        let outcome = call.as_mut().poll(&mut task_context);
15678        let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15679            panic!("one failed child must return a typed partial failure: {outcome:?}");
15680        };
15681        assert_eq!(failure.member_path, [1, 0]);
15682        assert_eq!(failure.group_id, "parallel-calls:1:3");
15683        assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15684        assert_eq!(
15685            failure
15686                .completed
15687                .iter()
15688                .map(|completion| completion.member_path.clone())
15689                .collect::<Vec<_>>(),
15690            [vec![0], vec![1, 1]]
15691        );
15692    }
15693
15694    #[test]
15695    fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15696        let paths = nested_parallel_paths();
15697        let history = vec![
15698            parallel_history_event(
15699                "ActivityScheduled",
15700                1,
15701                "activity_type",
15702                "first",
15703                paths[0].clone(),
15704                None,
15705            ),
15706            parallel_history_event(
15707                "ChildWorkflowScheduled",
15708                2,
15709                "child_workflow_type",
15710                "second",
15711                paths[1].clone(),
15712                None,
15713            ),
15714            parallel_history_event(
15715                "ActivityScheduled",
15716                3,
15717                "activity_type",
15718                "third",
15719                paths[2].clone(),
15720                None,
15721            ),
15722        ];
15723        for _restart in 0..2 {
15724            let ctx = workflow_context(history.clone());
15725            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15726            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15727            let outcome = call.as_mut().poll(&mut task_context);
15728            assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15729            assert!(ctx.take_commands().expect("commands").is_empty());
15730        }
15731    }
15732
15733    fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15734        vec![selection_group_entry(
15735            1,
15736            2,
15737            index,
15738            "activity",
15739            &SelectionMemberMetadata {
15740                key: SelectionKey::Name(key.to_string()),
15741                index,
15742                base_sequence: index as u64 + 1,
15743                size: 1,
15744                kind: "activity".to_string(),
15745            },
15746        )]
15747    }
15748
15749    fn selection_activity_event(
15750        event_type: &str,
15751        index: usize,
15752        key: &str,
15753        result: Option<Value>,
15754    ) -> HistoryEvent {
15755        let sequence = index as u64 + 1;
15756        let mut event = parallel_history_event(
15757            event_type,
15758            sequence,
15759            "activity_type",
15760            &format!("{key}-activity"),
15761            selection_path(index, key),
15762            result,
15763        );
15764        event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15765        event.raw.insert(
15766            "id".to_string(),
15767            json!(if event_type == "ActivityCompleted" {
15768                format!("event-{key}")
15769            } else {
15770                format!("{event_type}-{key}")
15771            }),
15772        );
15773        event
15774    }
15775
15776    fn selection_winner_marker() -> HistoryEvent {
15777        history_event(
15778            "SelectionResolved",
15779            json!({
15780                "selection_group_id": "select-calls:1:2",
15781                "selection_group_base_sequence": 1,
15782                "selection_group_size": 2,
15783                "member_key": "fast",
15784                "member_index": 1,
15785                "member_base_sequence": 2,
15786                "member_size": 1,
15787                "operation_kind": "activity",
15788                "operation_identity": "activity-fast",
15789                "outcome": "completed",
15790                "resolution_event_id": "event-fast",
15791                "resolution_event_type": "ActivityCompleted",
15792            }),
15793        )
15794    }
15795
15796    fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15797        ctx.select_keyed(vec![
15798            (
15799                "slow",
15800                ParallelOperation::activity_with_options(
15801                    "slow-activity",
15802                    ActivityOptions::new().task_queue("default"),
15803                    json!([]),
15804                ),
15805            ),
15806            (
15807                "fast",
15808                ParallelOperation::activity_with_options(
15809                    "fast-activity",
15810                    ActivityOptions::new().task_queue("default"),
15811                    json!([]),
15812                ),
15813            ),
15814        ])
15815    }
15816
15817    fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15818        let ctx = workflow_context(history);
15819        let mut call = Box::pin(keyed_activity_selection(&ctx));
15820        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15821        let selected = match call.as_mut().poll(&mut task_context) {
15822            Poll::Ready(Ok(selected)) => selected,
15823            Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15824            Poll::Pending => panic!("persisted selection winner must replay without pending"),
15825        };
15826        assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15827        assert_eq!(
15828            selected.value,
15829            Some(ParallelResult::Activity(json!("winner-value")))
15830        );
15831        let slow = selected
15832            .handle(&SelectionKey::Name("slow".to_string()))
15833            .expect("slow handle")
15834            .clone();
15835        let mut await_slow = Box::pin(slow.await_result());
15836        assert!(matches!(
15837            await_slow.as_mut().poll(&mut task_context),
15838            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15839        ));
15840        assert!(ctx.take_commands().expect("commands").is_empty());
15841    }
15842
15843    const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15844
15845    fn canonical_selection_history() -> Vec<HistoryEvent> {
15846        const FIXTURE: &[u8] =
15847            include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15848        assert_eq!(
15849            format!("{:x}", Sha256::digest(FIXTURE)),
15850            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15851        );
15852        let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15853
15854        serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15855    }
15856
15857    #[test]
15858    fn selection_fresh_process_entrypoint() {
15859        let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15860            return;
15861        };
15862        let persisted = fs::read(path).expect("persisted selection history");
15863        assert_eq!(
15864            format!("{:x}", Sha256::digest(&persisted)),
15865            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15866        );
15867        let fixture: Value =
15868            serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15869        let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15870            .expect("valid persisted selection history");
15871
15872        assert_persisted_selection_replay(history);
15873    }
15874
15875    #[test]
15876    fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15877        let ctx = workflow_context(Vec::new());
15878        let mut call = Box::pin(keyed_activity_selection(&ctx));
15879        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15880
15881        assert!(matches!(
15882            call.as_mut().poll(&mut task_context),
15883            Poll::Pending
15884        ));
15885        let commands = ctx.take_commands().expect("selection commands");
15886        assert_eq!(commands.len(), 2);
15887        assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15888        assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15889        assert!(commands.iter().all(|command| {
15890            command["parallel_group_id"] == json!("select-calls:1:2")
15891                && command["parallel_group_mode"] == json!("select")
15892        }));
15893    }
15894
15895    #[test]
15896    fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15897        let ctx = workflow_context(Vec::new());
15898        let mut invalid = Box::pin(ctx.select_keyed(vec![(
15899            "",
15900            ParallelOperation::activity("invalid", json!([])),
15901        )]));
15902        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15903        assert!(matches!(
15904            invalid.as_mut().poll(&mut task_context),
15905            Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15906                reason: "selection_key_invalid",
15907                ..
15908            })))
15909        ));
15910
15911        for invalid_key in [json!(""), json!(-1)] {
15912            let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15913            event.payload["selection_member_key"] = invalid_key.clone();
15914            event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15915            assert!(matches!(
15916                WorkflowState::new_with_identity(
15917                    vec![event],
15918                    None,
15919                    None,
15920                    "rust-workers".to_string(),
15921                    DEFAULT_CODEC.to_string(),
15922                    None,
15923                ),
15924                Err(Error::NonDeterministicReplay(_))
15925            ));
15926        }
15927    }
15928
15929    #[test]
15930    fn selection_preserves_valid_named_and_numeric_keys() {
15931        let ctx = workflow_context(Vec::new());
15932        let mut selection = Box::pin(ctx.select_keyed(vec![
15933            (
15934                SelectionKey::Index(0),
15935                ParallelOperation::activity("numeric", json!([])),
15936            ),
15937            (
15938                SelectionKey::Name("named".to_string()),
15939                ParallelOperation::timer(Duration::from_secs(1)),
15940            ),
15941        ]));
15942        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15943
15944        assert!(matches!(
15945            selection.as_mut().poll(&mut task_context),
15946            Poll::Pending
15947        ));
15948        let commands = ctx.take_commands().expect("selection commands");
15949        assert_eq!(commands[0]["selection_member_key"], json!(0));
15950        assert_eq!(commands[1]["selection_member_key"], json!("named"));
15951    }
15952
15953    #[test]
15954    fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15955        let history = canonical_selection_history();
15956        assert_persisted_selection_replay(history.clone());
15957
15958        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15959            .join("tests/fixtures/durable_selection_runtime_history.json");
15960        let output =
15961            ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15962                .args([
15963                    "--exact",
15964                    "tests::selection_fresh_process_entrypoint",
15965                    "--nocapture",
15966                ])
15967                .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15968                .output()
15969                .expect("run fresh selection replay process");
15970
15971        assert!(
15972            output.status.success(),
15973            "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15974            String::from_utf8_lossy(&output.stdout),
15975            String::from_utf8_lossy(&output.stderr),
15976        );
15977    }
15978
15979    #[test]
15980    fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15981        let mut history = canonical_selection_history();
15982        history.retain(|event| event.event_type != "SelectionResolved");
15983        let ctx = workflow_context(history);
15984        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15985        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15986
15987        assert!(matches!(
15988            selection.as_mut().poll(&mut task_context),
15989            Poll::Pending
15990        ));
15991        assert!(ctx.take_commands().expect("commands").is_empty());
15992        assert!(
15993            ctx.matched_recorded_pending()
15994                .expect("selection pending state"),
15995            "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15996        );
15997    }
15998
15999    #[test]
16000    fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
16001        for (terminal_event, predicate_satisfied, timeout_seconds) in [
16002            ("ConditionWaitSatisfied", true, None),
16003            ("ConditionWaitTimedOut", false, Some(0)),
16004        ] {
16005            let member = SelectionMemberMetadata {
16006                key: SelectionKey::Name("condition".to_string()),
16007                index: 0,
16008                base_sequence: 1,
16009                size: 1,
16010                kind: "condition".to_string(),
16011            };
16012            let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
16013            let mut payload = json!({
16014                "sequence": 1,
16015                "condition_wait_id": "condition-1",
16016                "condition_wait_occurrence_id": "rust:condition-wait:0",
16017                "condition_key": "ready",
16018                "condition_definition_fingerprint": "sha256:ready-v1",
16019                "parallel_group_path": path,
16020            });
16021            payload
16022                .as_object_mut()
16023                .expect("condition history payload")
16024                .extend(
16025                    serde_json::to_value(&path[0])
16026                        .expect("condition selection metadata")
16027                        .as_object()
16028                        .expect("condition selection metadata object")
16029                        .clone(),
16030                );
16031            if let Some(timeout_seconds) = timeout_seconds {
16032                payload["timeout_seconds"] = json!(timeout_seconds);
16033            }
16034            let history = vec![
16035                history_event("ConditionWaitOpened", payload.clone()),
16036                history_event(terminal_event, payload),
16037            ];
16038            let ctx = workflow_context(history);
16039            let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
16040            if timeout_seconds.is_some() {
16041                options = options.timeout(Duration::ZERO);
16042            }
16043            let mut selection = Box::pin(ctx.select_keyed(vec![(
16044                "condition",
16045                ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
16046            )]));
16047            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16048
16049            assert!(matches!(
16050                selection.as_mut().poll(&mut task_context),
16051                Poll::Pending
16052            ));
16053            assert!(ctx.take_commands().expect("commands").is_empty());
16054            assert!(
16055                ctx.matched_recorded_pending()
16056                    .expect("condition selection pending state"),
16057                "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
16058            );
16059        }
16060    }
16061
16062    #[test]
16063    fn selection_immediate_condition_members_open_a_durable_wait() {
16064        for predicate_satisfied in [true, false] {
16065            let ctx = workflow_context(Vec::new());
16066            let mut selection = Box::pin(ctx.select_keyed(vec![(
16067                "condition",
16068                ParallelOperation::condition(
16069                    ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
16070                    move || Ok(predicate_satisfied),
16071                ),
16072            )]));
16073            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16074
16075            assert!(matches!(
16076                selection.as_mut().poll(&mut task_context),
16077                Poll::Pending
16078            ));
16079            let commands = ctx.take_commands().expect("condition selection command");
16080            assert_eq!(commands.len(), 1);
16081            assert_eq!(commands[0]["type"], json!("open_condition_wait"));
16082            assert_eq!(commands[0]["timeout_seconds"], json!(0));
16083            assert_eq!(
16084                commands[0]["parallel_group_path"][0]["parallel_group_mode"],
16085                json!("select")
16086            );
16087        }
16088    }
16089
16090    #[test]
16091    fn selection_loser_cancellation_is_explicit_and_idempotent() {
16092        let history = vec![
16093            selection_activity_event("ActivityScheduled", 0, "slow", None),
16094            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16095            selection_winner_marker(),
16096        ];
16097        let ctx = workflow_context(history.clone());
16098        let mut call = Box::pin(keyed_activity_selection(&ctx));
16099        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16100        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
16101            panic!("winner must replay");
16102        };
16103        let slow = selected
16104            .handle(&SelectionKey::Name("slow".to_string()))
16105            .expect("slow handle")
16106            .clone();
16107        let mut cancel = Box::pin(slow.cancel());
16108        assert!(matches!(
16109            cancel.as_mut().poll(&mut task_context),
16110            Poll::Pending
16111        ));
16112        assert!(matches!(
16113            cancel.as_mut().poll(&mut task_context),
16114            Poll::Pending
16115        ));
16116        let commands = ctx.take_commands().expect("cancel command");
16117        assert_eq!(commands.len(), 1);
16118        assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
16119        assert_eq!(commands[0]["member_key"], json!("slow"));
16120
16121        let mut cancelled_history = history;
16122        cancelled_history.push(history_event(
16123            "SelectionOperationCancelled",
16124            json!({
16125                "selection_group_id": "select-calls:1:2",
16126                "member_key": "slow",
16127                "member_index": 0,
16128                "member_base_sequence": 1,
16129                "member_size": 1,
16130                "operation_kind": "activity",
16131                "operation_identity": "activity-slow",
16132                "cancelled_at": "2026-08-27T00:00:00Z",
16133            }),
16134        ));
16135        let replayed = workflow_context(cancelled_history);
16136        let mut call = Box::pin(keyed_activity_selection(&replayed));
16137        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
16138            panic!("winner must replay after cancellation");
16139        };
16140        let slow = selected
16141            .handle(&SelectionKey::Name("slow".to_string()))
16142            .expect("slow handle")
16143            .clone();
16144        let mut cancel = Box::pin(slow.cancel());
16145        assert!(matches!(
16146            cancel.as_mut().poll(&mut task_context),
16147            Poll::Ready(Ok(()))
16148        ));
16149        assert!(replayed.take_commands().expect("commands").is_empty());
16150    }
16151
16152    #[test]
16153    fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
16154        let base_history = vec![
16155            selection_activity_event("ActivityScheduled", 0, "slow", None),
16156            selection_activity_event("ActivityScheduled", 1, "fast", None),
16157            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16158            selection_winner_marker(),
16159        ];
16160        for (field, corrupt) in [
16161            ("member_key", json!("fast")),
16162            ("member_index", json!(1)),
16163            ("member_base_sequence", json!(3)),
16164            ("member_size", json!(2)),
16165            ("operation_kind", json!("timer")),
16166            ("operation_identity", json!("forged")),
16167        ] {
16168            let mut cancellation = json!({
16169                "selection_group_id": "select-calls:1:2",
16170                "member_key": "slow",
16171                "member_index": 0,
16172                "member_base_sequence": 1,
16173                "member_size": 1,
16174                "operation_kind": "activity",
16175                "operation_identity": "activity-slow",
16176            });
16177            cancellation[field] = corrupt;
16178            let mut history = base_history.clone();
16179            history.push(history_event("SelectionOperationCancelled", cancellation));
16180            let ctx = workflow_context(history);
16181            let mut selection = Box::pin(keyed_activity_selection(&ctx));
16182            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16183
16184            assert!(matches!(
16185                selection.as_mut().poll(&mut task_context),
16186                Poll::Ready(Err(Error::NonDeterministicReplay(_)))
16187            ));
16188        }
16189    }
16190
16191    #[test]
16192    fn selection_child_identity_prefers_the_durable_run_id() {
16193        let ctx = workflow_context(vec![history_event(
16194            "ChildWorkflowScheduled",
16195            json!({
16196                "sequence": 1,
16197                "child_workflow_type": "child",
16198                "child_workflow_instance_id": "child-instance",
16199                "child_workflow_run_id": "child-run",
16200            }),
16201        )]);
16202        let state = ctx.state.lock().expect("workflow state");
16203
16204        assert_eq!(
16205            selection_operation_identity(&state, "child", 1, 1),
16206            "child-run"
16207        );
16208    }
16209
16210    #[test]
16211    fn selection_activity_identity_requires_canonical_execution_id() {
16212        let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
16213        let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
16214        let mut fast_completed =
16215            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
16216        for event in [&mut fast_open, &mut fast_completed] {
16217            event
16218                .payload
16219                .as_object_mut()
16220                .expect("activity payload")
16221                .remove("activity_execution_id");
16222            event.payload["activity_id"] = json!("forged-activity-id");
16223        }
16224        let mut marker = selection_winner_marker();
16225        marker.payload["operation_identity"] = json!("forged-activity-id");
16226        let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
16227        let mut selection = Box::pin(keyed_activity_selection(&ctx));
16228        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16229
16230        assert!(matches!(
16231            selection.as_mut().poll(&mut task_context),
16232            Poll::Ready(Err(Error::NonDeterministicReplay(_)))
16233        ));
16234    }
16235
16236    #[test]
16237    fn selection_completion_before_cancellation_remains_awaitable() {
16238        let history = vec![
16239            selection_activity_event("ActivityScheduled", 0, "slow", None),
16240            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16241            selection_winner_marker(),
16242            selection_activity_event(
16243                "ActivityCompleted",
16244                0,
16245                "slow",
16246                Some(json!("completed-first")),
16247            ),
16248        ];
16249        let ctx = workflow_context(history);
16250        let mut selection = Box::pin(keyed_activity_selection(&ctx));
16251        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16252        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16253            panic!("winner must replay");
16254        };
16255        let slow = selected
16256            .handle(&SelectionKey::Name("slow".to_string()))
16257            .expect("slow handle")
16258            .clone();
16259        let mut cancel = Box::pin(slow.cancel());
16260        assert!(matches!(
16261            cancel.as_mut().poll(&mut task_context),
16262            Poll::Ready(Ok(()))
16263        ));
16264        let mut await_slow = Box::pin(slow.await_result());
16265        assert!(matches!(
16266            await_slow.as_mut().poll(&mut task_context),
16267            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
16268        ));
16269        let commands = ctx.take_commands().expect("commands");
16270        assert!(commands.is_empty());
16271    }
16272
16273    #[test]
16274    fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
16275        let nested_member = SelectionMemberMetadata {
16276            key: SelectionKey::Name("nested".to_string()),
16277            index: 0,
16278            base_sequence: 1,
16279            size: 2,
16280            kind: "group".to_string(),
16281        };
16282        let deadline_member = SelectionMemberMetadata {
16283            key: SelectionKey::Name("deadline".to_string()),
16284            index: 1,
16285            base_sequence: 3,
16286            size: 1,
16287            kind: "timer".to_string(),
16288        };
16289        let nested_paths = [
16290            vec![
16291                selection_group_entry(1, 3, 0, "mixed", &nested_member),
16292                parallel_group_entry(1, 2, 0, "activity"),
16293            ],
16294            vec![
16295                selection_group_entry(1, 3, 1, "mixed", &nested_member),
16296                parallel_group_entry(1, 2, 1, "activity"),
16297            ],
16298        ];
16299        let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
16300        let mut timer_fired = parallel_history_event(
16301            "TimerFired",
16302            3,
16303            "timer_id",
16304            "timer-3",
16305            deadline_path.clone(),
16306            None,
16307        );
16308        timer_fired.payload["delay_seconds"] = json!(0);
16309        timer_fired
16310            .raw
16311            .insert("id".to_string(), json!("timer-fired"));
16312        let mut timer_scheduled = parallel_history_event(
16313            "TimerScheduled",
16314            3,
16315            "timer_id",
16316            "timer-3",
16317            deadline_path,
16318            None,
16319        );
16320        timer_scheduled.payload["delay_seconds"] = json!(0);
16321        let history = vec![
16322            parallel_history_event(
16323                "ActivityScheduled",
16324                1,
16325                "activity_type",
16326                "nested-first",
16327                nested_paths[0].clone(),
16328                None,
16329            ),
16330            parallel_history_event(
16331                "ActivityScheduled",
16332                2,
16333                "activity_type",
16334                "nested-second",
16335                nested_paths[1].clone(),
16336                None,
16337            ),
16338            timer_scheduled,
16339            timer_fired,
16340            history_event(
16341                "SelectionResolved",
16342                json!({
16343                    "selection_group_id": "select-calls:1:3",
16344                    "selection_group_base_sequence": 1,
16345                    "selection_group_size": 3,
16346                    "member_key": "deadline",
16347                    "member_index": 1,
16348                    "member_base_sequence": 3,
16349                    "member_size": 1,
16350                    "operation_kind": "timer",
16351                    "operation_identity": "timer-3",
16352                    "outcome": "completed",
16353                    "resolution_event_id": "timer-fired",
16354                    "resolution_event_type": "TimerFired",
16355                }),
16356            ),
16357            parallel_history_event(
16358                "ActivityFailed",
16359                2,
16360                "activity_type",
16361                "nested-second",
16362                nested_paths[1].clone(),
16363                None,
16364            ),
16365        ];
16366        let ctx = workflow_context(history);
16367        let mut selection = Box::pin(ctx.select_keyed(vec![
16368            (
16369                "nested",
16370                ParallelOperation::group(vec![
16371                    ParallelOperation::activity("nested-first", json!([])),
16372                    ParallelOperation::activity("nested-second", json!([])),
16373                ]),
16374            ),
16375            ("deadline", ParallelOperation::timer(Duration::ZERO)),
16376        ]));
16377        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16378        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16379            panic!("deadline winner must replay");
16380        };
16381        let nested = selected
16382            .handle(&SelectionKey::Name("nested".to_string()))
16383            .expect("nested handle")
16384            .clone();
16385        let mut cancel = Box::pin(nested.cancel());
16386        assert!(matches!(
16387            cancel.as_mut().poll(&mut task_context),
16388            Poll::Ready(Ok(()))
16389        ));
16390        let mut await_nested = Box::pin(nested.await_result());
16391
16392        assert!(matches!(
16393            await_nested.as_mut().poll(&mut task_context),
16394            Poll::Ready(Err(Error::ActivityFailed(_)))
16395        ));
16396        assert!(ctx.take_commands().expect("commands").is_empty());
16397    }
16398
16399    #[test]
16400    fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16401        let ctx = workflow_context(Vec::new());
16402        let mut call = Box::pin(ctx.select(vec![
16403            ParallelOperation::child_workflow(
16404                "child",
16405                ChildWorkflowOptions::new("children"),
16406                json!([]),
16407            ),
16408            ParallelOperation::timer(Duration::from_secs(30)),
16409            ParallelOperation::signal("approval"),
16410            ParallelOperation::condition(
16411                ConditionWaitOptions::new("ready", "sha256:ready"),
16412                || Ok(false),
16413            ),
16414            ParallelOperation::group(vec![
16415                ParallelOperation::activity("nested-one", json!([])),
16416                ParallelOperation::activity("nested-two", json!([])),
16417            ]),
16418        ]));
16419        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16420        assert!(matches!(
16421            call.as_mut().poll(&mut task_context),
16422            Poll::Pending
16423        ));
16424        let commands = ctx.take_commands().expect("selection commands");
16425        assert_eq!(
16426            commands
16427                .iter()
16428                .map(|command| command["type"].as_str().unwrap_or_default())
16429                .collect::<Vec<_>>(),
16430            [
16431                "start_child_workflow",
16432                "start_timer",
16433                "open_signal_wait",
16434                "open_condition_wait",
16435                "schedule_activity",
16436                "schedule_activity",
16437            ]
16438        );
16439        assert!(commands.iter().all(|command| {
16440            command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16441        }));
16442        assert_eq!(
16443            commands[4]["parallel_group_path"].as_array().map(Vec::len),
16444            Some(2)
16445        );
16446        assert_eq!(
16447            commands[4]["parallel_group_path"][0]["selection_member_kind"],
16448            json!("group")
16449        );
16450        assert_eq!(
16451            commands[5]["parallel_group_path"][0]["selection_member_kind"],
16452            json!("group")
16453        );
16454
16455        let one_leaf_ctx = workflow_context(Vec::new());
16456        let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16457            ParallelOperation::activity("nested-only", json!([])),
16458        ])]));
16459        assert!(matches!(
16460            one_leaf.as_mut().poll(&mut task_context),
16461            Poll::Pending
16462        ));
16463        let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16464        assert_eq!(one_leaf_commands.len(), 1);
16465        assert_eq!(
16466            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16467            json!("group")
16468        );
16469        assert_eq!(
16470            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16471            json!(1)
16472        );
16473    }
16474
16475    async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16476        let mut saga = ctx.saga();
16477        let outcome = async {
16478            let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16479            saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16480            let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16481            saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16482            ctx.activity("trip.charge", json!([])).await?;
16483            Ok(json!({"status": "booked"}))
16484        }
16485        .await;
16486        saga.finish(outcome).await
16487    }
16488
16489    fn saga_activity(
16490        event_type: &str,
16491        sequence: u64,
16492        activity_type: &str,
16493        result: Option<Value>,
16494    ) -> HistoryEvent {
16495        let mut payload = json!({
16496            "sequence": sequence,
16497            "activity_type": activity_type,
16498            "message": format!("{activity_type} failed"),
16499            "exception_type": "PlannedFailure",
16500            "non_retryable": true,
16501        });
16502        if let Some(result) = result {
16503            payload["result"] = fixture_envelope(result);
16504        }
16505        history_event(event_type, payload)
16506    }
16507
16508    #[test]
16509    fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16510        let completed_hotel_compensation = saga_activity(
16511            "ActivityCompleted",
16512            4,
16513            "trip.cancel-hotel",
16514            Some(Value::Null),
16515        );
16516        let history = vec![
16517            saga_activity(
16518                "ActivityCompleted",
16519                1,
16520                "trip.reserve-flight",
16521                Some(json!("flight-1")),
16522            ),
16523            saga_activity(
16524                "ActivityCompleted",
16525                2,
16526                "trip.reserve-hotel",
16527                Some(json!("hotel-1")),
16528            ),
16529            saga_activity("ActivityFailed", 3, "trip.charge", None),
16530            completed_hotel_compensation.clone(),
16531            completed_hotel_compensation,
16532        ];
16533
16534        for _restart in 0..2 {
16535            let ctx = workflow_context(history.clone());
16536            let mut future = Box::pin(trip_saga(ctx.clone()));
16537            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16538            assert!(matches!(
16539                future.as_mut().poll(&mut task_context),
16540                Poll::Pending
16541            ));
16542            let commands = ctx.take_commands().expect("compensation command");
16543            assert_eq!(commands.len(), 1);
16544            assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16545        }
16546    }
16547
16548    #[test]
16549    fn saga_compensation_failure_preserves_both_typed_failures() {
16550        let history = vec![
16551            saga_activity(
16552                "ActivityCompleted",
16553                1,
16554                "trip.reserve-flight",
16555                Some(json!("flight-1")),
16556            ),
16557            saga_activity(
16558                "ActivityCompleted",
16559                2,
16560                "trip.reserve-hotel",
16561                Some(json!("hotel-1")),
16562            ),
16563            saga_activity("ActivityFailed", 3, "trip.charge", None),
16564            saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16565        ];
16566        let ctx = workflow_context(history);
16567        let mut future = Box::pin(trip_saga(ctx));
16568        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16569        let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16570            future.as_mut().poll(&mut task_context)
16571        else {
16572            panic!("compensation failure must remain structured");
16573        };
16574        assert!(matches!(
16575            *failure.initiating_failure,
16576            Error::ActivityFailed(_)
16577        ));
16578        assert!(matches!(
16579            *failure.compensation_failure,
16580            Error::ActivityFailed(_)
16581        ));
16582        assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16583        assert_eq!(failure.compensation_registration_order, 2);
16584    }
16585
16586    #[test]
16587    fn saga_compensates_cooperative_cancellation() {
16588        let ctx = workflow_context(vec![saga_activity(
16589            "ActivityCompleted",
16590            1,
16591            "trip.reserve-flight",
16592            Some(json!("flight-1")),
16593        )]);
16594        ctx.state.lock().expect("state").cancel_requested = true;
16595        let run = {
16596            let ctx = ctx.clone();
16597            async move {
16598                let mut saga = ctx.saga();
16599                let outcome = async {
16600                    let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16601                    saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16602                    ctx.throw_if_cancellation_requested()?;
16603                    Ok(json!("unexpected"))
16604                }
16605                .await;
16606                saga.finish(outcome).await
16607            }
16608        };
16609        let mut future = Box::pin(run);
16610        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16611        assert!(matches!(
16612            future.as_mut().poll(&mut task_context),
16613            Poll::Pending
16614        ));
16615        let commands = ctx.take_commands().expect("cancellation compensation");
16616        assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16617    }
16618
16619    fn workflow_task(
16620        workflow_type: &str,
16621        history_events: Vec<HistoryEvent>,
16622        payload_codec: &str,
16623    ) -> WorkflowTask {
16624        WorkflowTask {
16625            task_id: format!("wft-{workflow_type}"),
16626            workflow_command_id: None,
16627            workflow_id: Some(format!("wf-{workflow_type}")),
16628            run_id: Some(format!("run-{workflow_type}")),
16629            workflow_type: workflow_type.to_string(),
16630            cancel_requested: false,
16631            payload_codec: payload_codec.to_string(),
16632            arguments: Some(
16633                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16634            ),
16635            total_history_events: Some(history_events.len() as u64),
16636            history_size_bytes: None,
16637            continue_as_new_recommended: None,
16638            history_budget_pressure: None,
16639            history_events,
16640            next_history_page_token: None,
16641            workflow_task_attempt: 1,
16642            workflow_signal_id: None,
16643            signal_name: None,
16644            signal_arguments: None,
16645            workflow_update_id: None,
16646            update_name: None,
16647            lease_owner: Some("rust-worker".to_string()),
16648        }
16649    }
16650
16651    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16652    struct SideEffectProbe {
16653        request_id: String,
16654        attempt: u32,
16655    }
16656
16657    #[test]
16658    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16659        let calls = AtomicUsize::new(0);
16660        let ctx = workflow_context(Vec::new());
16661        let value = ctx
16662            .side_effect(|| {
16663                calls.fetch_add(1, Ordering::SeqCst);
16664                SideEffectProbe {
16665                    request_id: "request-42".to_string(),
16666                    attempt: 3,
16667                }
16668            })
16669            .expect("first side effect");
16670        assert_eq!(value.attempt, 3);
16671        assert_eq!(calls.load(Ordering::SeqCst), 1);
16672        let commands = ctx.take_commands().expect("commands");
16673        assert_eq!(commands.len(), 1);
16674        assert_eq!(commands[0]["type"], "record_side_effect");
16675        assert_eq!(
16676            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16677            serde_json::to_value(&value).expect("value")
16678        );
16679
16680        let replay = workflow_context(vec![history_event(
16681            "SideEffectRecorded",
16682            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16683        )]);
16684        let replayed: SideEffectProbe = replay
16685            .side_effect(|| {
16686                calls.fetch_add(1, Ordering::SeqCst);
16687                panic!("committed side-effect callbacks must not run during replay")
16688            })
16689            .expect("replayed side effect");
16690        assert_eq!(replayed, value);
16691        assert_eq!(calls.load(Ordering::SeqCst), 1);
16692        assert!(replay.take_commands().expect("commands").is_empty());
16693        replay.ensure_history_consumed().expect("history consumed");
16694    }
16695
16696    #[test]
16697    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16698        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16699        let value = ctx
16700            .side_effect(|| SideEffectProbe {
16701                request_id: "avro-request".to_string(),
16702                attempt: 1,
16703            })
16704            .expect("Avro side effect");
16705        let uuid = ctx.uuid_v4().expect("deterministic UUID");
16706        let commands = ctx.take_commands().expect("commands");
16707        assert_eq!(commands.len(), 2);
16708        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16709        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16710        assert_eq!(
16711            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16712            serde_json::to_value(&value).expect("value")
16713        );
16714
16715        let replay = workflow_context_with_codec(
16716            vec![
16717                history_event(
16718                    "SideEffectRecorded",
16719                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16720                ),
16721                history_event(
16722                    "SideEffectRecorded",
16723                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16724                ),
16725            ],
16726            DEFAULT_CODEC,
16727        );
16728        let replayed: SideEffectProbe = replay
16729            .side_effect(|| panic!("Avro callback must not run"))
16730            .expect("replayed Avro value");
16731        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16732        assert_eq!(replayed, value);
16733        assert_eq!(replayed_uuid, uuid);
16734        assert!(replay.take_commands().expect("commands").is_empty());
16735    }
16736
16737    #[test]
16738    fn typed_side_effect_replay_preserves_bytes_and_maps() {
16739        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16740        let value = ctx
16741            .side_effect_avro_value(typed_fidelity_probe)
16742            .expect("typed side effect");
16743        let commands = ctx.take_commands().expect("side-effect command");
16744        assert_eq!(
16745            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16746                .expect("recorded side effect"),
16747            value
16748        );
16749
16750        let replay = workflow_context_with_codec(
16751            vec![history_event(
16752                "SideEffectRecorded",
16753                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16754            )],
16755            DEFAULT_CODEC,
16756        );
16757        assert_eq!(
16758            replay
16759                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16760                .expect("replayed typed side effect"),
16761            value
16762        );
16763    }
16764
16765    #[test]
16766    fn ordered_side_effects_share_the_durable_command_stream() {
16767        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16768        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16769        let ctx = workflow_context(vec![
16770            history_event(
16771                "SideEffectRecorded",
16772                json!({"sequence": 1, "result": first}),
16773            ),
16774            history_event(
16775                "SideEffectRecorded",
16776                json!({"sequence": 2, "result": second}),
16777            ),
16778        ]);
16779        let first: String = ctx
16780            .side_effect(|| panic!("first callback must not run"))
16781            .expect("first replay");
16782        let second: i32 = ctx
16783            .side_effect(|| panic!("second callback must not run"))
16784            .expect("second replay");
16785        assert_eq!(first, "first");
16786        assert_eq!(second, 29);
16787        ctx.ensure_history_consumed().expect("ordered history");
16788
16789        let reordered = workflow_context(vec![history_event(
16790            "VersionMarkerRecorded",
16791            json!({
16792                "sequence": 1,
16793                "change_id": "before-side-effect",
16794                "version": 1,
16795                "min_supported": 1,
16796                "max_supported": 1,
16797            }),
16798        )]);
16799        let error = reordered
16800            .side_effect(|| "new".to_string())
16801            .expect_err("command reordering must fail");
16802        assert!(matches!(
16803            error,
16804            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16805                if reason == "recorded_command_mismatch"
16806        ));
16807    }
16808
16809    #[test]
16810    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16811        let ctx = workflow_context(Vec::new());
16812        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16813        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16814        assert!(ctx.patched("new-search").expect("patch"));
16815        ctx.deprecate_patch("new-search").expect("deprecate patch");
16816        let commands = ctx.take_commands().expect("commands");
16817        assert_eq!(commands.len(), 2);
16818        assert_eq!(commands[0]["type"], "record_version_marker");
16819        assert_eq!(commands[0]["version"], 2);
16820        assert_eq!(commands[1]["change_id"], "new-search");
16821
16822        let replay = workflow_context(vec![history_event(
16823            "VersionMarkerRecorded",
16824            json!({
16825                "sequence": 1,
16826                "change_id": "checkout-v2",
16827                "version": 2,
16828                "min_supported": 1,
16829                "max_supported": 2,
16830            }),
16831        )]);
16832        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16833        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16834        assert!(replay.take_commands().expect("commands").is_empty());
16835        replay.ensure_history_consumed().expect("history consumed");
16836    }
16837
16838    #[test]
16839    fn version_markers_reject_incompatible_or_malformed_history() {
16840        let incompatible = workflow_context(vec![history_event(
16841            "VersionMarkerRecorded",
16842            json!({
16843                "sequence": 1,
16844                "change_id": "checkout-v2",
16845                "version": 1,
16846                "min_supported": 1,
16847                "max_supported": 2,
16848            }),
16849        )]);
16850        let error = incompatible
16851            .get_version("checkout-v2", 2, 3)
16852            .expect_err("old version is unsupported");
16853        assert!(matches!(
16854            error,
16855            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16856                if reason == "version_marker_incompatible_range"
16857        ));
16858
16859        for (history, reason) in [
16860            (
16861                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16862                "side_effect_result_missing",
16863            ),
16864            (
16865                vec![history_event(
16866                    "SideEffectRecorded",
16867                    json!({
16868                        "sequence": 1,
16869                        "result": {"codec": "avro", "blob": "not-base64"},
16870                    }),
16871                )],
16872                "side_effect_payload_incompatible",
16873            ),
16874            (
16875                vec![history_event(
16876                    "SideEffectRecorded",
16877                    json!({"sequence": 1, "result": {"unwrapped": true}}),
16878                )],
16879                "side_effect_payload_malformed",
16880            ),
16881            (
16882                vec![history_event(
16883                    "VersionMarkerRecorded",
16884                    json!({
16885                        "sequence": 1,
16886                        "change_id": "change",
16887                        "version": 1,
16888                        "min_supported": 2,
16889                        "max_supported": 1,
16890                    }),
16891                )],
16892                "version_marker_history_range_invalid",
16893            ),
16894        ] {
16895            let error = WorkflowState::new(
16896                history,
16897                "rust-workers".to_string(),
16898                DEFAULT_CODEC.to_string(),
16899                None,
16900            )
16901            .expect_err("malformed history must fail");
16902            assert!(matches!(
16903                error,
16904                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16905                    if actual == reason
16906            ));
16907        }
16908    }
16909
16910    #[test]
16911    fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16912        let history = vec![history_event(
16913            "SearchAttributesUpserted",
16914            json!({
16915                "sequence": 1,
16916                "attributes": {"customer_tier": "gold"},
16917                "attribute_types": {"customer_tier": "keyword"},
16918                "merged": {"customer_tier": "gold"}
16919            }),
16920        )];
16921
16922        let matching = workflow_context(history.clone());
16923        matching
16924            .upsert_search_attributes(
16925                SearchAttributeUpdate::new()
16926                    .keyword("customer_tier", "gold")
16927                    .expect("keyword update"),
16928            )
16929            .expect("matching typed update must replay");
16930        matching
16931            .ensure_history_consumed()
16932            .expect("history consumed");
16933
16934        let changed_type = workflow_context(history.clone());
16935        let error = changed_type
16936            .upsert_search_attributes(
16937                SearchAttributeUpdate::new()
16938                    .string("customer_tier", "gold")
16939                    .expect("string update"),
16940            )
16941            .expect_err("same JSON value with a different declaration must be nondeterministic");
16942        let Error::NonDeterministicReplay(failure) = error else {
16943            panic!("typed identity drift must be a replay failure");
16944        };
16945        assert_eq!(failure.reason, "search_attribute_type_mismatch");
16946        assert_eq!(failure.sequence, Some(1));
16947
16948        let changed_value = workflow_context(history);
16949        let error = changed_value
16950            .upsert_search_attributes(
16951                SearchAttributeUpdate::new()
16952                    .keyword("customer_tier", "platinum")
16953                    .expect("keyword update"),
16954            )
16955            .expect_err("changed values must be nondeterministic");
16956        let Error::NonDeterministicReplay(failure) = error else {
16957            panic!("value drift must be a replay failure");
16958        };
16959        assert_eq!(failure.reason, "search_attribute_value_mismatch");
16960    }
16961
16962    #[test]
16963    fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16964        let history = vec![history_event(
16965            "SearchAttributesUpserted",
16966            json!({
16967                "sequence": 1,
16968                "attributes": {"customer_tier": "gold"},
16969                "merged": {"customer_tier": "gold"}
16970            }),
16971        )];
16972
16973        for update in [
16974            SearchAttributeUpdate::new()
16975                .keyword("customer_tier", "gold")
16976                .expect("keyword update"),
16977            SearchAttributeUpdate::new()
16978                .string("customer_tier", "gold")
16979                .expect("string update"),
16980        ] {
16981            let restarted = workflow_context(history.clone());
16982            restarted
16983                .upsert_search_attributes(update)
16984                .expect("legacy history constrains values but has unknown type identity");
16985            restarted
16986                .ensure_history_consumed()
16987                .expect("history consumed");
16988        }
16989    }
16990
16991    #[test]
16992    fn search_attribute_command_emits_canonical_types() {
16993        let ctx = workflow_context(Vec::new());
16994        ctx.upsert_search_attributes(
16995            SearchAttributeUpdate::new()
16996                .keyword("customer_tier", "gold")
16997                .expect("keyword update")
16998                .int("attempts", 3)
16999                .expect("int update")
17000                .delete("obsolete")
17001                .expect("delete update"),
17002        )
17003        .expect("valid search attributes");
17004
17005        assert_eq!(
17006            ctx.take_commands().expect("commands"),
17007            vec![json!({
17008                "type": "upsert_search_attributes",
17009                "attributes": {
17010                    "attempts": 3,
17011                    "customer_tier": "gold",
17012                    "obsolete": null
17013                },
17014                "attribute_types": {
17015                    "attempts": "int",
17016                    "customer_tier": "keyword"
17017                }
17018            })]
17019        );
17020    }
17021
17022    #[test]
17023    fn duplicate_side_effects_and_version_markers_are_rejected() {
17024        let duplicate_side_effect = WorkflowState::new(
17025            vec![
17026                history_event(
17027                    "SideEffectRecorded",
17028                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
17029                ),
17030                history_event(
17031                    "SideEffectRecorded",
17032                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
17033                ),
17034            ],
17035            "rust-workers".to_string(),
17036            DEFAULT_CODEC.to_string(),
17037            None,
17038        )
17039        .expect_err("duplicate side effect");
17040        assert!(matches!(
17041            duplicate_side_effect,
17042            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17043                if reason == "duplicate_side_effect_record"
17044        ));
17045
17046        let marker = |sequence| {
17047            history_event(
17048                "VersionMarkerRecorded",
17049                json!({
17050                    "sequence": sequence,
17051                    "change_id": "same-change",
17052                    "version": 1,
17053                    "min_supported": 1,
17054                    "max_supported": 1,
17055                }),
17056            )
17057        };
17058        let duplicate_marker = WorkflowState::new(
17059            vec![marker(1), marker(3)],
17060            "rust-workers".to_string(),
17061            DEFAULT_CODEC.to_string(),
17062            None,
17063        )
17064        .expect_err("duplicate marker");
17065        assert!(matches!(
17066            duplicate_marker,
17067            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17068                if reason == "duplicate_version_marker"
17069        ));
17070    }
17071
17072    #[test]
17073    fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
17074        let mut state = WorkflowState::new(
17075            Vec::new(),
17076            "rust-workers".to_string(),
17077            DEFAULT_CODEC.to_string(),
17078            None,
17079        )
17080        .expect("workflow state");
17081        state.workflow_command_identity = "command-7".to_string();
17082        let context = WorkflowContext {
17083            state: Arc::new(Mutex::new(state)),
17084        };
17085        let item =
17086            WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
17087
17088        context
17089            .append_workflow_stream("output", &[item], Some(10))
17090            .expect("append command");
17091        context
17092            .error_workflow_stream("output", "producer failed", None)
17093            .expect("error command");
17094        let commands = context.take_commands().expect("commands");
17095
17096        assert_eq!(commands[0]["type"], "record_side_effect");
17097        assert_eq!(
17098            commands[0]["workflow_stream"]["command_identity"],
17099            "command-7"
17100        );
17101        assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
17102        assert_eq!(
17103            commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
17104            "dw-stream:command-7:0:0"
17105        );
17106        assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
17107
17108        let recorded = history_event(
17109            "SideEffectRecorded",
17110            json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
17111        );
17112        let mut replay_state = WorkflowState::new(
17113            vec![recorded],
17114            "rust-workers".to_string(),
17115            DEFAULT_CODEC.to_string(),
17116            None,
17117        )
17118        .expect("replay state");
17119        replay_state.workflow_command_identity = "command-7".to_string();
17120        let replay_context = WorkflowContext {
17121            state: Arc::new(Mutex::new(replay_state)),
17122        };
17123        replay_context
17124            .append_workflow_stream(
17125                "output",
17126                &[WorkflowStreamAppendItem::from_reference(
17127                    "s3://bucket/item.avro",
17128                )],
17129                Some(10),
17130            )
17131            .expect("replayed append");
17132        assert!(replay_context
17133            .take_commands()
17134            .expect("replayed commands")
17135            .is_empty());
17136    }
17137
17138    #[test]
17139    fn workflow_stream_authoring_requires_server_durable_command_identity() {
17140        let context = workflow_context(Vec::new());
17141        let error = context
17142            .append_workflow_stream(
17143                "output",
17144                &[WorkflowStreamAppendItem::from_reference(
17145                    "s3://bucket/item.avro",
17146                )],
17147                None,
17148            )
17149            .expect_err("stream append without durable command identity must fail closed");
17150
17151        assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
17152        assert!(context.take_commands().expect("commands").is_empty());
17153    }
17154
17155    #[test]
17156    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
17157        fn worker(calls: Arc<AtomicUsize>) -> Worker {
17158            let client = Client::new("http://127.0.0.1:8080").expect("client");
17159            let mut worker = Worker::new(client, "rust-workers");
17160            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
17161                let calls = Arc::clone(&calls);
17162                async move {
17163                    let captured = ctx.side_effect(|| {
17164                        calls.fetch_add(1, Ordering::SeqCst);
17165                        "captured-once".to_string()
17166                    })?;
17167                    let version = ctx.get_version("cold-restart", 1, 2)?;
17168                    Ok(json!({"captured": captured, "version": version}))
17169                }
17170            });
17171            worker
17172        }
17173
17174        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
17175            WorkflowTask {
17176                task_id: "wft-side-effect-version".to_string(),
17177                workflow_command_id: None,
17178                workflow_id: Some("wf-side-effect-version".to_string()),
17179                run_id: Some("run-side-effect-version".to_string()),
17180                workflow_type: "rust.side-effect-version".to_string(),
17181                cancel_requested: false,
17182                payload_codec: DEFAULT_CODEC.to_string(),
17183                arguments: Some(
17184                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
17185                ),
17186                history_events,
17187                total_history_events: None,
17188                history_size_bytes: None,
17189                continue_as_new_recommended: None,
17190                history_budget_pressure: None,
17191                next_history_page_token: None,
17192                workflow_task_attempt: 1,
17193                workflow_signal_id: None,
17194                signal_name: None,
17195                signal_arguments: None,
17196                workflow_update_id: None,
17197                update_name: None,
17198                lease_owner: Some("rust-worker".to_string()),
17199            }
17200        }
17201
17202        let calls = Arc::new(AtomicUsize::new(0));
17203        let initial = worker(Arc::clone(&calls))
17204            .execute_workflow_task(task(Vec::new()))
17205            .expect("initial execution");
17206        assert_eq!(
17207            initial
17208                .iter()
17209                .map(|command| &command["type"])
17210                .collect::<Vec<_>>(),
17211            vec![
17212                "record_side_effect",
17213                "record_version_marker",
17214                "complete_workflow"
17215            ]
17216        );
17217        assert_eq!(calls.load(Ordering::SeqCst), 1);
17218
17219        let restarted = worker(Arc::clone(&calls));
17220        let replayed = restarted
17221            .execute_workflow_task(task(vec![
17222                history_event(
17223                    "SideEffectRecorded",
17224                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
17225                ),
17226                history_event(
17227                    "VersionMarkerRecorded",
17228                    json!({
17229                        "sequence": 2,
17230                        "change_id": "cold-restart",
17231                        "version": 2,
17232                        "min_supported": 1,
17233                        "max_supported": 2,
17234                    }),
17235                ),
17236            ]))
17237            .expect("cold replay");
17238        assert_eq!(replayed.len(), 1);
17239        assert_eq!(replayed[0]["type"], "complete_workflow");
17240        assert_eq!(calls.load(Ordering::SeqCst), 1);
17241    }
17242
17243    #[test]
17244    fn side_effect_replay_rejects_changed_rust_value_type() {
17245        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
17246        let ctx = workflow_context(vec![history_event(
17247            "SideEffectRecorded",
17248            json!({"sequence": 1, "result": result}),
17249        )]);
17250        let error = ctx
17251            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
17252            .expect_err("changed type must fail replay");
17253        assert!(matches!(
17254            error,
17255            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17256                if reason == "side_effect_type_mismatch"
17257        ));
17258    }
17259
17260    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
17261        vec![
17262            history_event(
17263                "ActivityScheduled",
17264                json!({
17265                    "sequence": 1,
17266                    "activity_type": "flaky",
17267                    "activity_execution_id": "act-1",
17268                    "activity": {
17269                        "id": "act-1",
17270                        "sequence": 1,
17271                        "type": "flaky",
17272                        "queue": "critical-activities",
17273                        "execution_mode": null,
17274                        "retry_policy": {
17275                            "snapshot_version": 1,
17276                            "max_attempts": 3,
17277                            "backoff_seconds": [2, 4],
17278                            "start_to_close_timeout": 30,
17279                            "schedule_to_start_timeout": 5,
17280                            "schedule_to_close_timeout": 90,
17281                            "heartbeat_timeout": 10,
17282                            "non_retryable_error_types": ["PermanentError"]
17283                        }
17284                    }
17285                }),
17286            ),
17287            history_event(
17288                "ActivityStarted",
17289                json!({
17290                    "sequence": 1,
17291                    "activity_type": "flaky",
17292                    "activity_execution_id": "act-1",
17293                    "activity_attempt_id": "attempt-1",
17294                    "attempt_number": 1
17295                }),
17296            ),
17297            history_event(
17298                "ActivityRetryScheduled",
17299                json!({
17300                    "sequence": 1,
17301                    "activity_type": "flaky",
17302                    "activity_execution_id": "act-1",
17303                    "activity_attempt_id": "attempt-1",
17304                    "attempt_number": 1,
17305                    "retry_after_attempt": 1,
17306                    "retry_backoff_seconds": 2,
17307                    "failure_category": "activity",
17308                    "exception_type": "TransientError"
17309                }),
17310            ),
17311            history_event(
17312                "ActivityStarted",
17313                json!({
17314                    "sequence": 1,
17315                    "activity_type": "flaky",
17316                    "activity_execution_id": "act-1",
17317                    "activity_attempt_id": "attempt-2",
17318                    "attempt_number": 2
17319                }),
17320            ),
17321            history_event(
17322                "ActivityCompleted",
17323                json!({
17324                    "sequence": 1,
17325                    "activity_type": "flaky",
17326                    "activity_execution_id": "act-1",
17327                    "activity_attempt_id": "attempt-2",
17328                    "attempt_number": 2,
17329                    "payload_codec": DEFAULT_CODEC,
17330                    "result": fixture_envelope(json!({"status":"recovered"}))
17331                }),
17332            ),
17333        ]
17334    }
17335
17336    fn retry_activity_options() -> ActivityOptions {
17337        ActivityOptions::new()
17338            .task_queue("critical-activities")
17339            .retry_policy(
17340                ActivityRetryPolicy::new(3)
17341                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
17342                    .non_retryable_error_type("PermanentError"),
17343            )
17344            .start_to_close_timeout(Duration::from_secs(30))
17345            .schedule_to_start_timeout(Duration::from_secs(5))
17346            .schedule_to_close_timeout(Duration::from_secs(90))
17347            .heartbeat_timeout(Duration::from_secs(10))
17348    }
17349
17350    #[test]
17351    fn fixed_avro_value_round_trips_json_values() {
17352        let value = json!({"greeting": "hello", "count": 3, "ok": true});
17353        let envelope = PayloadEnvelope::avro(&value).expect("encode");
17354        assert_eq!(envelope.codec, DEFAULT_CODEC);
17355        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
17356    }
17357
17358    #[tokio::test]
17359    async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
17360        let client = Client::new("http://127.0.0.1:8080").expect("client");
17361        let mut worker = Worker::new(client, "rust-workers");
17362        worker.register_typed_workflow(
17363            "typed.contract.workflow",
17364            |_ctx, input: TypedContract| async move { Ok(input) },
17365        );
17366        worker.register_typed_activity(
17367            "typed.contract.activity",
17368            |_ctx, input: TypedContract| async move { Ok(input) },
17369        );
17370
17371        let expected = typed_contract();
17372        let arguments = AvroValue::Array(vec![
17373            AvroValue::from_serialize(&expected).expect("typed request")
17374        ]);
17375        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
17376        let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
17377        workflow.arguments = Some(envelope.clone());
17378        let commands = worker
17379            .execute_workflow_task(workflow)
17380            .expect("typed workflow task");
17381        let workflow_result: TypedContract =
17382            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17383                .expect("workflow result envelope")
17384                .deserialize()
17385                .expect("workflow result type");
17386        assert_eq!(workflow_result, expected);
17387
17388        let activity = ActivityTask {
17389            task_id: "typed-contract-activity".to_string(),
17390            activity_attempt_id: Some("typed-contract-attempt".to_string()),
17391            attempt_id: None,
17392            activity_type: "typed.contract.activity".to_string(),
17393            payload_codec: DEFAULT_CODEC.to_string(),
17394            arguments: Some(envelope),
17395            attempt_number: 1,
17396            lease_owner: Some("rust-worker".to_string()),
17397        };
17398        let activity_result: TypedContract = worker
17399            .execute_activity_task(activity)
17400            .await
17401            .expect("typed activity task")
17402            .deserialize()
17403            .expect("activity result type");
17404        assert_eq!(activity_result, expected);
17405    }
17406
17407    #[tokio::test]
17408    async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17409        let client = Client::new("http://127.0.0.1:8080").expect("client");
17410        let mut worker = Worker::new(client, "rust-workers");
17411        worker.register_typed_workflow(
17412            "typed.shape.workflow",
17413            |_ctx, input: TypedContract| async move { Ok(input) },
17414        );
17415        worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17416            Ok(f64::NAN)
17417        });
17418
17419        let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17420        workflow.arguments = Some(
17421            encode_typed_envelope(
17422                &AvroValue::Array(vec![
17423                    AvroValue::String("first".to_string()),
17424                    AvroValue::String("second".to_string()),
17425                ]),
17426                DEFAULT_CODEC,
17427            )
17428            .expect("malformed typed arguments"),
17429        );
17430        let commands = worker
17431            .execute_workflow_task(workflow)
17432            .expect("shape mismatch becomes a workflow failure");
17433        let message = commands[0]["message"].as_str().expect("failure message");
17434        assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17435        assert!(message.contains(type_name::<TypedContract>()));
17436        assert!(message.contains("task carried 2 arguments"));
17437
17438        let activity = ActivityTask {
17439            task_id: "typed-unsupported-activity".to_string(),
17440            activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17441            attempt_id: None,
17442            activity_type: "typed.unsupported.activity".to_string(),
17443            payload_codec: DEFAULT_CODEC.to_string(),
17444            arguments: Some(
17445                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17446                    .expect("unit arguments"),
17447            ),
17448            attempt_number: 1,
17449            lease_owner: Some("rust-worker".to_string()),
17450        };
17451        let Error::HandlerType {
17452            handler_kind,
17453            handler_name,
17454            value_kind,
17455            rust_type,
17456            message,
17457        } = worker
17458            .execute_activity_task(activity)
17459            .await
17460            .expect_err("non-finite handler output must fail")
17461        else {
17462            panic!("expected contextual handler type failure");
17463        };
17464        assert_eq!(handler_kind, HandlerKind::Activity);
17465        assert_eq!(handler_name, "typed.unsupported.activity");
17466        assert_eq!(value_kind, HandlerValueKind::Result);
17467        assert_eq!(rust_type, type_name::<f64>());
17468        assert!(message.contains("non_finite_float"));
17469    }
17470
17471    #[tokio::test]
17472    async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17473        #[derive(Clone, Default)]
17474        struct State {
17475            observed: Option<TypedContract>,
17476        }
17477
17478        let client = Client::new("http://127.0.0.1:8080").expect("client");
17479        let mut worker = Worker::new(client, "rust-workers");
17480        worker.register_typed_replayed_workflow(
17481            "typed.contract.replayed",
17482            State::default,
17483            |ctx, input: TypedContract, state| async move {
17484                let result: TypedContract =
17485                    ctx.activity_typed("typed.contract.activity", input).await?;
17486                state.update(|current| current.observed = Some(result.clone()))?;
17487                Ok(result)
17488            },
17489        );
17490        worker.register_replayed_query::<State, _, _>(
17491            "typed.contract.replayed",
17492            "observed",
17493            |_ctx, state, _args| async move {
17494                Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17495            },
17496        );
17497
17498        let expected = typed_contract();
17499        let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17500        let workflow_arguments =
17501            encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17502                .expect("workflow arguments");
17503        let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17504        let task = QueryTask {
17505            query_task_id: "typed-replay-query".to_string(),
17506            query_task_attempt: 1,
17507            lease_owner: Some("rust-worker".to_string()),
17508            workflow_id: Some("typed-replay".to_string()),
17509            run_id: Some("typed-replay-run".to_string()),
17510            workflow_type: "typed.contract.replayed".to_string(),
17511            query_name: "observed".to_string(),
17512            payload_codec: DEFAULT_CODEC.to_string(),
17513            workflow_arguments: Some(workflow_arguments),
17514            query_arguments: Some(
17515                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17516                    .expect("query arguments"),
17517            ),
17518            history_events: vec![
17519                history_event(
17520                    "ActivityScheduled",
17521                    json!({
17522                        "sequence": 1,
17523                        "activity_type": "typed.contract.activity"
17524                    }),
17525                ),
17526                history_event(
17527                    "ActivityCompleted",
17528                    json!({
17529                        "sequence": 1,
17530                        "activity_type": "typed.contract.activity",
17531                        "payload_codec": DEFAULT_CODEC,
17532                        "result": result
17533                    }),
17534                ),
17535            ],
17536            history_export: None,
17537            run_status: Some("completed".to_string()),
17538        };
17539
17540        assert_eq!(
17541            worker
17542                .execute_query_task(task)
17543                .await
17544                .expect("typed replay query")
17545                .deserialize::<i64>()
17546                .expect("query result"),
17547            expected.signed
17548        );
17549    }
17550
17551    #[tokio::test]
17552    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17553        let client = Client::new("http://127.0.0.1:8080").expect("client");
17554        let mut worker = Worker::new(client, "rust-workers");
17555        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17556        worker
17557            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17558        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17559            Ok(input)
17560        });
17561        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17562            Ok(input)
17563        });
17564        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17565            Ok(AvroValue::Array(
17566                ctx.wait_signal_avro_value("changed").await?,
17567            ))
17568        });
17569
17570        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17571        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17572
17573        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17574        workflow.arguments = Some(envelope.clone());
17575        let commands = worker
17576            .execute_workflow_task(workflow)
17577            .expect("typed workflow task");
17578        assert_eq!(commands[0]["type"], "complete_workflow");
17579        assert_eq!(
17580            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17581                .expect("typed workflow result"),
17582            arguments
17583        );
17584
17585        let activity = ActivityTask {
17586            task_id: "activity-typed".to_string(),
17587            activity_attempt_id: Some("attempt-typed".to_string()),
17588            attempt_id: None,
17589            activity_type: "typed.activity".to_string(),
17590            payload_codec: DEFAULT_CODEC.to_string(),
17591            arguments: Some(envelope.clone()),
17592            attempt_number: 1,
17593            lease_owner: Some("rust-worker".to_string()),
17594        };
17595        assert_eq!(
17596            worker
17597                .execute_activity_task(activity)
17598                .await
17599                .expect("typed activity result"),
17600            arguments
17601        );
17602
17603        let query = QueryTask {
17604            query_task_id: "query-typed".to_string(),
17605            query_task_attempt: 1,
17606            lease_owner: Some("rust-worker".to_string()),
17607            workflow_id: Some("typed-1".to_string()),
17608            run_id: Some("run-typed".to_string()),
17609            workflow_type: "typed.echo".to_string(),
17610            query_name: "inspect".to_string(),
17611            payload_codec: DEFAULT_CODEC.to_string(),
17612            workflow_arguments: Some(
17613                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17614                    .expect("workflow input"),
17615            ),
17616            query_arguments: Some(envelope.clone()),
17617            history_events: Vec::new(),
17618            history_export: None,
17619            run_status: Some("running".to_string()),
17620        };
17621        assert_eq!(
17622            worker
17623                .execute_query_task(query)
17624                .await
17625                .expect("typed query result"),
17626            arguments
17627        );
17628
17629        let mut update = workflow_task(
17630            "typed.echo",
17631            vec![history_event(
17632                "UpdateAccepted",
17633                json!({
17634                    "update_id": "update-typed",
17635                    "update_name": "replace",
17636                    "arguments": envelope.clone(),
17637                }),
17638            )],
17639            DEFAULT_CODEC,
17640        );
17641        update.workflow_update_id = Some("update-typed".to_string());
17642        update.update_name = Some("replace".to_string());
17643        let commands = worker
17644            .execute_workflow_task(update)
17645            .expect("typed update task");
17646        assert_eq!(commands[0]["type"], "complete_update");
17647        assert_eq!(
17648            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17649                .expect("typed update result"),
17650            arguments
17651        );
17652
17653        let mut signal = workflow_task(
17654            "typed.signal",
17655            vec![history_event(
17656                "SignalReceived",
17657                json!({
17658                    "signal_id": "signal-typed",
17659                    "signal_name": "changed",
17660                    "arguments": envelope.clone(),
17661                }),
17662            )],
17663            DEFAULT_CODEC,
17664        );
17665        signal.workflow_signal_id = Some("signal-typed".to_string());
17666        signal.signal_name = Some("changed".to_string());
17667        signal.signal_arguments = Some(envelope);
17668        let commands = worker
17669            .execute_workflow_task(signal)
17670            .expect("typed signal resume");
17671        assert_eq!(
17672            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17673                .expect("typed signal result"),
17674            arguments
17675        );
17676    }
17677
17678    #[tokio::test]
17679    async fn typed_helpers_never_parse_json_inspection_projection() {
17680        let collision_values = projection_collision_probe();
17681        let expected = AvroValue::Array(collision_values.clone());
17682        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17683
17684        let activity_context = workflow_context_with_codec(
17685            vec![history_event(
17686                "ActivityCompleted",
17687                json!({
17688                    "sequence": 1,
17689                    "activity_type": "collision.activity",
17690                    "payload_codec": DEFAULT_CODEC,
17691                    "result": envelope.clone(),
17692                }),
17693            )],
17694            DEFAULT_CODEC,
17695        );
17696        assert_eq!(
17697            activity_context
17698                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17699                .await
17700                .expect("typed activity collision result"),
17701            expected
17702        );
17703
17704        let signal_context = workflow_context_with_codec(
17705            vec![
17706                history_event(
17707                    "SignalWaitOpened",
17708                    json!({"sequence": 1, "signal_name": "collision"}),
17709                ),
17710                history_event(
17711                    "SignalApplied",
17712                    json!({
17713                        "sequence": 1,
17714                        "signal_name": "collision",
17715                        "payload_codec": DEFAULT_CODEC,
17716                        "value": envelope.clone(),
17717                    }),
17718                ),
17719            ],
17720            DEFAULT_CODEC,
17721        );
17722        assert_eq!(
17723            signal_context
17724                .wait_signal_avro_value("collision")
17725                .await
17726                .expect("typed signal collision arguments"),
17727            collision_values
17728        );
17729
17730        let child_context = workflow_context_with_codec(
17731            vec![
17732                history_event(
17733                    "ChildWorkflowScheduled",
17734                    json!({
17735                        "sequence": 1,
17736                        "child_workflow_instance_id": "collision-child",
17737                        "child_workflow_run_id": "collision-run",
17738                        "child_workflow_type": "collision.child",
17739                    }),
17740                ),
17741                history_event(
17742                    "ChildRunCompleted",
17743                    json!({
17744                        "sequence": 1,
17745                        "child_workflow_instance_id": "collision-child",
17746                        "child_workflow_run_id": "collision-run",
17747                        "child_workflow_type": "collision.child",
17748                        "payload_codec": DEFAULT_CODEC,
17749                        "result": envelope,
17750                    }),
17751                ),
17752            ],
17753            DEFAULT_CODEC,
17754        );
17755        let child = child_context
17756            .start_child_workflow_avro_value(
17757                "collision.child",
17758                ChildWorkflowOptions::new("collision-workers"),
17759                AvroValue::Array(Vec::new()),
17760            )
17761            .await
17762            .expect("typed child collision result");
17763        assert_eq!(child.result, expected);
17764    }
17765
17766    #[tokio::test]
17767    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17768        let client = Client::new("http://127.0.0.1:8080").expect("client");
17769        let mut worker = Worker::new(client, "rust-workers");
17770        worker.register_replayed_workflow_avro_value(
17771            "typed.replayed",
17772            || (),
17773            |_ctx, input, _state| async move { Ok(input) },
17774        );
17775        worker.register_replayed_query_avro_value::<(), _, _>(
17776            "typed.replayed",
17777            "inspect",
17778            |ctx, _state, args| async move {
17779                let mut signals = ctx.signals_avro_value("collision");
17780                let signal = signals
17781                    .pop()
17782                    .map(AvroValue::Array)
17783                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17784                Ok(AvroValue::Array(vec![
17785                    ctx.workflow_input_avro_value().clone(),
17786                    signal,
17787                    args,
17788                ]))
17789            },
17790        );
17791        let arguments = AvroValue::Array(projection_collision_probe());
17792        let signal_arguments =
17793            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17794        let task = QueryTask {
17795            query_task_id: "query-typed-replay".to_string(),
17796            query_task_attempt: 1,
17797            lease_owner: Some("rust-worker".to_string()),
17798            workflow_id: Some("typed-replay".to_string()),
17799            run_id: Some("run-typed-replay".to_string()),
17800            workflow_type: "typed.replayed".to_string(),
17801            query_name: "inspect".to_string(),
17802            payload_codec: DEFAULT_CODEC.to_string(),
17803            workflow_arguments: Some(
17804                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17805            ),
17806            query_arguments: Some(
17807                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17808            ),
17809            history_events: vec![history_event(
17810                "SignalReceived",
17811                json!({
17812                    "signal_id": "collision-signal",
17813                    "signal_name": "collision",
17814                    "workflow_sequence": 1,
17815                    "payload_codec": DEFAULT_CODEC,
17816                    "arguments": signal_arguments,
17817                }),
17818            )],
17819            history_export: None,
17820            run_status: Some("completed".to_string()),
17821        };
17822
17823        assert_eq!(
17824            worker
17825                .execute_query_task(task)
17826                .await
17827                .expect("typed replay query"),
17828            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17829        );
17830    }
17831
17832    #[test]
17833    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17834        let value = BTreeMap::from([(1_i32, "integer key")]);
17835        let error = PayloadEnvelope::avro(&value)
17836            .expect_err("integer map keys must fail")
17837            .to_string();
17838
17839        assert!(error.contains("invalid_map_key"));
17840    }
17841
17842    #[test]
17843    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17844        let envelope = PayloadEnvelope {
17845            codec: "json".to_string(),
17846            blob: r#"{"greeting":"hello"}"#.to_string(),
17847        };
17848
17849        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17850        let diagnostic = error.to_string();
17851        assert!(diagnostic.contains("unsupported_payload_codec"));
17852        assert!(diagnostic.contains("codec=\"avro\""));
17853        assert!(diagnostic.contains("HTTP document transport"));
17854    }
17855
17856    #[test]
17857    fn untagged_json_payload_value_fails_closed() {
17858        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17859            .expect_err("untagged JSON payload values must fail");
17860        let diagnostic = error.to_string();
17861        assert!(diagnostic.contains("unsupported_payload_codec"));
17862        assert!(diagnostic.contains("untagged durable payload"));
17863        assert!(diagnostic.contains("HTTP document transport"));
17864    }
17865
17866    #[test]
17867    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17868        let envelope = PayloadEnvelope {
17869            codec: DEFAULT_CODEC.to_string(),
17870            blob: BASE64.encode([0x01]),
17871        };
17872
17873        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17874        assert!(error.to_string().contains("invalid_payload_framing"));
17875    }
17876
17877    #[tokio::test]
17878    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17879        let server = MockWorkerServer::start();
17880        let client = Client::builder(server.base_url())
17881            .timeout(Duration::from_secs(2))
17882            .build()
17883            .expect("client");
17884        let invalid_commands = [
17885            json!({
17886                "type": "complete_workflow",
17887                "result": {"codec": "json", "blob": null}
17888            }),
17889            json!({
17890                "type": "schedule_activity",
17891                "arguments": {"codec": "yaml", "blob": "ignored"}
17892            }),
17893            json!({
17894                "type": "start_child_workflow",
17895                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17896            }),
17897            json!({"type": "continue_as_new", "arguments": []}),
17898            json!({"type": "complete_update"}),
17899            json!({"type": "record_side_effect", "result": null}),
17900            json!({
17901                "type": "start_service_operation",
17902                "payload_codec": DEFAULT_CODEC,
17903                "request_payload": "raw-avro-bytes"
17904            }),
17905        ];
17906
17907        for command in invalid_commands {
17908            let error = client
17909                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17910                .await
17911                .expect_err("invalid durable payload must fail locally");
17912            let diagnostic = error.to_string();
17913            assert!(
17914                diagnostic.contains("unsupported_payload_codec")
17915                    || diagnostic.contains("invalid_payload_envelope")
17916                    || diagnostic.contains("untagged durable payload"),
17917                "unexpected validation diagnostic: {diagnostic}"
17918            );
17919        }
17920
17921        assert_eq!(
17922            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17923            0,
17924            "invalid command payloads must not reach HTTP transport"
17925        );
17926    }
17927
17928    #[test]
17929    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17930        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17931        let commands = [
17932            json!({"type": "complete_workflow", "result": envelope.clone()}),
17933            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17934            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17935            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17936            json!({"type": "complete_update", "result": envelope.clone()}),
17937            json!({"type": "record_side_effect", "result": envelope.clone()}),
17938            json!({
17939                "type": "start_service_operation",
17940                "payload_codec": DEFAULT_CODEC,
17941                "request_payload": envelope.clone()
17942            }),
17943            json!({
17944                "type": "complete_workflow",
17945                "result": envelope,
17946                "metadata": {
17947                    "codec": "json",
17948                    "payload_codec": "customer-codec",
17949                    "result": {"codec": "yaml", "blob": null}
17950                }
17951            }),
17952        ];
17953
17954        validate_workflow_task_commands(&commands)
17955            .expect("customer metadata must not become a protocol codec declaration");
17956    }
17957
17958    #[test]
17959    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17960        assert_eq!(
17961            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17962            AvroValue::Array(Vec::new())
17963        );
17964        assert_eq!(
17965            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17966            AvroValue::Array(Vec::new())
17967        );
17968
17969        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17970        signal.signal_name = Some("empty-signal".to_string());
17971        signal.signal_arguments = None;
17972        let decoded = decode_resume_signal(&signal)
17973            .expect("valid Avro signal")
17974            .expect("named signal resumes the workflow");
17975        assert!(decoded.arguments.is_empty());
17976    }
17977
17978    #[tokio::test]
17979    async fn malformed_task_level_codecs_become_pre_handler_failures() {
17980        let client = Client::new("http://127.0.0.1:8080").expect("client");
17981        let mut worker = Worker::new(client, "rust-workers");
17982        let handler_calls = Arc::new(AtomicUsize::new(0));
17983
17984        let calls = Arc::clone(&handler_calls);
17985        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17986            calls.fetch_add(1, Ordering::SeqCst);
17987            async move { Ok(Value::Null) }
17988        });
17989        let calls = Arc::clone(&handler_calls);
17990        worker.register_activity("codec.activity", move |_ctx, _args| {
17991            calls.fetch_add(1, Ordering::SeqCst);
17992            async move { Ok(Value::Null) }
17993        });
17994        let calls = Arc::clone(&handler_calls);
17995        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17996            calls.fetch_add(1, Ordering::SeqCst);
17997            async move { Ok(Value::Null) }
17998        });
17999
18000        let mut failures = Vec::new();
18001        for codec_case in [
18002            InvalidTaskPayloadCodec::Missing,
18003            InvalidTaskPayloadCodec::Null,
18004            InvalidTaskPayloadCodec::NonString,
18005        ] {
18006            let mut workflow = json!({
18007                "task_id": format!("workflow-{}", codec_case.label()),
18008                "workflow_type": "codec.workflow"
18009            });
18010            codec_case.apply(&mut workflow);
18011            match serde_json::from_value::<WorkflowTask>(workflow) {
18012                Ok(task) => match worker.execute_workflow_task(task) {
18013                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
18014                    outcome => failures.push(format!(
18015                        "workflow {} codec returned {outcome:?}",
18016                        codec_case.label()
18017                    )),
18018                },
18019                Err(error) => failures.push(format!(
18020                    "workflow {} codec failed transport deserialization: {error}",
18021                    codec_case.label()
18022                )),
18023            }
18024
18025            let mut activity = json!({
18026                "task_id": format!("activity-{}", codec_case.label()),
18027                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
18028                "activity_type": "codec.activity",
18029                "attempt_number": 1
18030            });
18031            codec_case.apply(&mut activity);
18032            match serde_json::from_value::<ActivityTask>(activity) {
18033                Ok(task) => match worker.execute_activity_task(task).await {
18034                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
18035                    outcome => failures.push(format!(
18036                        "activity {} codec returned {outcome:?}",
18037                        codec_case.label()
18038                    )),
18039                },
18040                Err(error) => failures.push(format!(
18041                    "activity {} codec failed transport deserialization: {error}",
18042                    codec_case.label()
18043                )),
18044            }
18045
18046            let mut query = json!({
18047                "query_task_id": format!("query-{}", codec_case.label()),
18048                "workflow_type": "codec.workflow",
18049                "query_name": "known"
18050            });
18051            codec_case.apply(&mut query);
18052            match serde_json::from_value::<QueryTask>(query) {
18053                Ok(task) => match worker.execute_query_task(task).await {
18054                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
18055                    outcome => failures.push(format!(
18056                        "query {} codec returned {outcome:?}",
18057                        codec_case.label()
18058                    )),
18059                },
18060                Err(error) => failures.push(format!(
18061                    "query {} codec failed transport deserialization: {error}",
18062                    codec_case.label()
18063                )),
18064            }
18065        }
18066
18067        assert!(failures.is_empty(), "{}", failures.join("\n"));
18068        assert_eq!(
18069            handler_calls.load(Ordering::SeqCst),
18070            0,
18071            "invalid task codecs must not invoke a handler"
18072        );
18073    }
18074
18075    #[tokio::test]
18076    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
18077        for codec_case in [
18078            InvalidTaskPayloadCodec::Missing,
18079            InvalidTaskPayloadCodec::Null,
18080            InvalidTaskPayloadCodec::NonString,
18081        ] {
18082            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
18083            let client = Client::builder(server.base_url())
18084                .timeout(Duration::from_secs(2))
18085                .build()
18086                .expect("client");
18087            let mut worker = Worker::new(client, "rust-workers")
18088                .worker_id("codec-worker")
18089                .poll_timeout(Duration::from_millis(10));
18090            let handler_calls = Arc::new(AtomicUsize::new(0));
18091
18092            let calls = Arc::clone(&handler_calls);
18093            worker.register_workflow("codec.workflow", move |_ctx, _args| {
18094                calls.fetch_add(1, Ordering::SeqCst);
18095                async move { Ok(Value::Null) }
18096            });
18097            let calls = Arc::clone(&handler_calls);
18098            worker.register_activity("codec.activity", move |_ctx, _args| {
18099                calls.fetch_add(1, Ordering::SeqCst);
18100                async move { Ok(Value::Null) }
18101            });
18102            let calls = Arc::clone(&handler_calls);
18103            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
18104                calls.fetch_add(1, Ordering::SeqCst);
18105                async move { Ok(Value::Null) }
18106            });
18107
18108            assert_eq!(
18109                worker.run_once().await.expect("invalid tasks are settled"),
18110                3,
18111                "all {} codec tasks must be handled",
18112                codec_case.label()
18113            );
18114            assert_eq!(
18115                handler_calls.load(Ordering::SeqCst),
18116                0,
18117                "{} task codecs must fail before every handler",
18118                codec_case.label()
18119            );
18120
18121            for path in [
18122                "/api/worker/workflow-tasks/codec-workflow/fail",
18123                "/api/worker/activity-tasks/codec-activity/fail",
18124                "/api/worker/query-tasks/codec-query/fail",
18125            ] {
18126                let body = server.request_body(path);
18127                assert!(
18128                    body["failure"]["message"]
18129                        .as_str()
18130                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
18131                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
18132                    codec_case.label()
18133                );
18134            }
18135            assert_eq!(
18136                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
18137                    ["reason"],
18138                "query_payload_decode_failed"
18139            );
18140            for path in [
18141                "/api/worker/workflow-tasks/codec-workflow/complete",
18142                "/api/worker/activity-tasks/codec-activity/complete",
18143                "/api/worker/query-tasks/codec-query/complete",
18144            ] {
18145                assert_eq!(
18146                    server.request_count(path),
18147                    0,
18148                    "invalid {} codec task reached {path}",
18149                    codec_case.label()
18150                );
18151            }
18152        }
18153    }
18154
18155    #[tokio::test]
18156    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
18157        let client = Client::new("http://127.0.0.1:8080").expect("client");
18158        let mut worker = Worker::new(client, "rust-workers");
18159        let handler_calls = Arc::new(AtomicUsize::new(0));
18160
18161        let calls = Arc::clone(&handler_calls);
18162        worker.register_workflow("codec.workflow", move |_ctx, _args| {
18163            calls.fetch_add(1, Ordering::SeqCst);
18164            async move { Ok(Value::Null) }
18165        });
18166        let calls = Arc::clone(&handler_calls);
18167        worker.register_activity("codec.activity", move |_ctx, _args| {
18168            calls.fetch_add(1, Ordering::SeqCst);
18169            async move { Ok(Value::Null) }
18170        });
18171        let calls = Arc::clone(&handler_calls);
18172        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
18173            calls.fetch_add(1, Ordering::SeqCst);
18174            async move { Ok(Value::Null) }
18175        });
18176        let calls = Arc::clone(&handler_calls);
18177        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
18178            calls.fetch_add(1, Ordering::SeqCst);
18179            async move { Ok(Value::Null) }
18180        });
18181
18182        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18183        workflow.payload_codec = "json".to_string();
18184        workflow.arguments = None;
18185        let error = worker
18186            .execute_workflow_task(workflow)
18187            .expect_err("task codec must be checked before workflow invocation");
18188        assert!(error.to_string().contains("unsupported_payload_codec"));
18189
18190        let activity = ActivityTask {
18191            task_id: "activity-invalid-codec".to_string(),
18192            activity_attempt_id: None,
18193            attempt_id: None,
18194            activity_type: "codec.activity".to_string(),
18195            payload_codec: "unknown".to_string(),
18196            arguments: None,
18197            attempt_number: 1,
18198            lease_owner: None,
18199        };
18200        let error = worker
18201            .execute_activity_task(activity)
18202            .await
18203            .expect_err("task codec must be checked before activity invocation");
18204        assert!(error.to_string().contains("unsupported_payload_codec"));
18205
18206        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18207        update.workflow_update_id = Some("update-invalid-codec".to_string());
18208        update.update_name = Some("known".to_string());
18209        update.history_events.push(history_event(
18210            "UpdateAccepted",
18211            json!({
18212                "update_id": "update-invalid-codec",
18213                "update_name": "known",
18214                "arguments": {"codec": "json", "blob": null}
18215            }),
18216        ));
18217        let error = worker
18218            .execute_workflow_task(update)
18219            .expect_err("nested update codec must be checked before handler lookup");
18220        assert!(error.to_string().contains("unsupported_payload_codec"));
18221
18222        let query: QueryTask = serde_json::from_value(json!({
18223            "query_task_id": "query-invalid-codec",
18224            "workflow_type": "codec.workflow",
18225            "query_name": "known",
18226            "payload_codec": DEFAULT_CODEC,
18227            "workflow_arguments": null,
18228            "query_arguments": null,
18229            "history_export": {
18230                "payloads": {"codec": DEFAULT_CODEC},
18231                "signals": [{
18232                    "name": "empty",
18233                    "payload_codec": "json",
18234                    "arguments": null
18235                }]
18236            }
18237        }))
18238        .expect("query task");
18239        let failure = worker
18240            .execute_query_task(query)
18241            .await
18242            .expect_err("exported signal codec must be checked before query invocation");
18243        assert_eq!(failure.reason, "query_payload_decode_failed");
18244        assert!(failure.message.contains("unsupported_payload_codec"));
18245
18246        let exported_history: QueryTask = serde_json::from_value(json!({
18247            "query_task_id": "query-invalid-history-codec",
18248            "workflow_type": "codec.workflow",
18249            "query_name": "known",
18250            "payload_codec": DEFAULT_CODEC,
18251            "history_export": {
18252                "payloads": {"codec": DEFAULT_CODEC},
18253                "history_events": [{
18254                    "type": "ActivityCompleted",
18255                    "payload": {"payload_codec": "unknown", "result": null}
18256                }]
18257            }
18258        }))
18259        .expect("query task");
18260        let failure = worker
18261            .execute_query_task(exported_history)
18262            .await
18263            .expect_err("exported history codec must be checked before query invocation");
18264        assert_eq!(failure.reason, "query_payload_decode_failed");
18265        assert!(failure.message.contains("unsupported_payload_codec"));
18266        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
18267
18268        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
18269        unknown_workflow.arguments = None;
18270        unknown_workflow.history_events.push(history_event(
18271            "SignalReceived",
18272            json!({
18273                "signal_name": "empty",
18274                "payload_codec": "json",
18275                "arguments": null
18276            }),
18277        ));
18278        let error = worker
18279            .execute_workflow_task(unknown_workflow)
18280            .expect_err("history codec must precede unknown workflow outcome");
18281        assert!(error.to_string().contains("unsupported_payload_codec"));
18282
18283        let unknown_activity = ActivityTask {
18284            task_id: "activity-unknown".to_string(),
18285            activity_attempt_id: None,
18286            attempt_id: None,
18287            activity_type: "missing".to_string(),
18288            payload_codec: "json".to_string(),
18289            arguments: None,
18290            attempt_number: 1,
18291            lease_owner: None,
18292        };
18293        let error = worker
18294            .execute_activity_task(unknown_activity)
18295            .await
18296            .expect_err("codec must precede unknown activity outcome");
18297        assert!(error.to_string().contains("unsupported_payload_codec"));
18298
18299        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18300        unknown_update.payload_codec = "json".to_string();
18301        unknown_update.arguments = None;
18302        unknown_update.workflow_update_id = Some("update-unknown".to_string());
18303        unknown_update.update_name = Some("missing".to_string());
18304        let error = worker
18305            .execute_workflow_task(unknown_update)
18306            .expect_err("codec must precede fail_update shortcut");
18307        assert!(error.to_string().contains("unsupported_payload_codec"));
18308
18309        let unknown_query: QueryTask = serde_json::from_value(json!({
18310            "query_task_id": "query-unknown",
18311            "workflow_type": "missing",
18312            "query_name": "missing",
18313            "payload_codec": "json",
18314            "workflow_arguments": null,
18315            "query_arguments": null
18316        }))
18317        .expect("query task");
18318        let failure = worker
18319            .execute_query_task(unknown_query)
18320            .await
18321            .expect_err("codec must precede unknown query outcome");
18322        assert_eq!(failure.reason, "query_payload_decode_failed");
18323        assert!(failure.message.contains("unsupported_payload_codec"));
18324    }
18325
18326    #[tokio::test]
18327    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
18328        let client = Client::new("http://127.0.0.1:8080").expect("client");
18329        let worker = Worker::new(client, "rust-workers");
18330
18331        for event_type in ["SignalReceived", "SignalApplied"] {
18332            for (payload_field, codec) in [
18333                ("value", "json"),
18334                ("input", "unknown"),
18335                ("arguments", "json"),
18336            ] {
18337                let payload = json!({
18338                    "signal_name": "empty",
18339                    payload_field: {"codec": codec, "blob": null}
18340                });
18341                let workflow = workflow_task(
18342                    "missing",
18343                    vec![history_event(event_type, payload.clone())],
18344                    DEFAULT_CODEC,
18345                );
18346                let error = worker
18347                    .execute_workflow_task(workflow)
18348                    .expect_err("signal payload codec must precede unknown workflow outcome");
18349                assert!(
18350                    error.to_string().contains("unsupported_payload_codec"),
18351                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
18352                );
18353
18354                let query: QueryTask = serde_json::from_value(json!({
18355                    "query_task_id": format!("query-{event_type}-{payload_field}"),
18356                    "workflow_type": "missing",
18357                    "query_name": "missing",
18358                    "payload_codec": DEFAULT_CODEC,
18359                    "workflow_arguments": null,
18360                    "query_arguments": null,
18361                    "history_events": [{
18362                        "event_type": event_type,
18363                        "payload": payload
18364                    }]
18365                }))
18366                .expect("query task");
18367                let failure = worker
18368                    .execute_query_task(query)
18369                    .await
18370                    .expect_err("signal payload codec must precede unknown query outcome");
18371                assert_eq!(
18372                    failure.reason, "query_payload_decode_failed",
18373                    "{event_type}.{payload_field} returned an unrelated query outcome"
18374                );
18375                assert!(
18376                    failure.message.contains("unsupported_payload_codec"),
18377                    "{event_type}.{payload_field} returned an unrelated query error: {}",
18378                    failure.message
18379                );
18380            }
18381        }
18382    }
18383
18384    #[test]
18385    fn workflow_context_schedules_activity_until_completion_is_in_history() {
18386        let ctx = WorkflowContext {
18387            state: Arc::new(Mutex::new(
18388                WorkflowState::new_with_identity(
18389                    Vec::new(),
18390                    Some("wf-parent".to_string()),
18391                    Some("run-parent".to_string()),
18392                    "rust-workers".to_string(),
18393                    DEFAULT_CODEC.to_string(),
18394                    None,
18395                )
18396                .expect("workflow state"),
18397            )),
18398        };
18399
18400        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18401        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18402        assert!(matches!(
18403            call.as_mut().poll(&mut task_context),
18404            Poll::Pending
18405        ));
18406
18407        let commands = ctx.take_commands().expect("commands");
18408        assert_eq!(commands[0]["type"], "schedule_activity");
18409        assert_eq!(commands[0]["activity_type"], "hello.activity");
18410    }
18411
18412    #[test]
18413    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18414        let ctx = workflow_context(Vec::new());
18415        let options = ActivityOptions::new()
18416            .task_queue("payments")
18417            .retry_policy(
18418                ActivityRetryPolicy::new(4)
18419                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18420                    .non_retryable_error_type("ValidationError"),
18421            )
18422            .start_to_close_timeout(Duration::from_secs(120))
18423            .schedule_to_start_timeout(Duration::from_secs(10))
18424            .schedule_to_close_timeout(Duration::from_secs(300))
18425            .heartbeat_timeout(Duration::from_secs(15));
18426        let mut call = Box::pin(ctx.activity_with_options(
18427            "charge-card",
18428            options,
18429            json!([{"order_id": "o-1"}]),
18430        ));
18431        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18432
18433        assert!(matches!(
18434            call.as_mut().poll(&mut task_context),
18435            Poll::Pending
18436        ));
18437        assert!(matches!(
18438            call.as_mut().poll(&mut task_context),
18439            Poll::Pending
18440        ));
18441
18442        let commands = ctx.take_commands().expect("activity command");
18443        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18444        assert_eq!(commands[0]["queue"], "payments");
18445        assert_eq!(
18446            commands[0]["retry_policy"],
18447            json!({
18448                "max_attempts": 4,
18449                "backoff_seconds": [1, 3, 9],
18450                "non_retryable_error_types": ["ValidationError"],
18451            })
18452        );
18453        assert_eq!(commands[0]["start_to_close_timeout"], 120);
18454        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18455        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18456        assert_eq!(commands[0]["heartbeat_timeout"], 15);
18457    }
18458
18459    #[test]
18460    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18461        let ctx = workflow_context(Vec::new());
18462        let options = ActivityOptions::new().retry_policy(
18463            ActivityRetryPolicy::new(3)
18464                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18465        );
18466        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18467        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18468
18469        assert!(matches!(
18470            call.as_mut().poll(&mut task_context),
18471            Poll::Pending
18472        ));
18473        assert_eq!(
18474            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18475            json!([1, 2])
18476        );
18477    }
18478
18479    #[test]
18480    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18481        let cases = [
18482            (
18483                ActivityOptions::new().task_queue("  "),
18484                ActivityOptionsErrorKind::EmptyTaskQueue,
18485            ),
18486            (
18487                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18488                ActivityOptionsErrorKind::EmptyRetryPolicy,
18489            ),
18490            (
18491                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18492                ActivityOptionsErrorKind::InvalidMaxAttempts,
18493            ),
18494            (
18495                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18496                    max_attempts: None,
18497                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18498                    non_retryable_error_types: Vec::new(),
18499                }),
18500                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18501            ),
18502            (
18503                ActivityOptions::new().retry_policy(
18504                    ActivityRetryPolicy::new(2)
18505                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18506                ),
18507                ActivityOptionsErrorKind::TooManyBackoffIntervals,
18508            ),
18509            (
18510                ActivityOptions::new().retry_policy(
18511                    ActivityRetryPolicy::new(2).exponential_backoff(
18512                        Duration::from_secs(1),
18513                        0,
18514                        None,
18515                    ),
18516                ),
18517                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18518            ),
18519            (
18520                ActivityOptions::new()
18521                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
18522                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18523            ),
18524            (
18525                ActivityOptions::new().retry_policy(
18526                    ActivityRetryPolicy::new(10_002).exponential_backoff(
18527                        Duration::from_secs(1),
18528                        1,
18529                        None,
18530                    ),
18531                ),
18532                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18533            ),
18534            (
18535                ActivityOptions::new().retry_policy(
18536                    ActivityRetryPolicy::new(2)
18537                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18538                ),
18539                ActivityOptionsErrorKind::BackoffOverflow,
18540            ),
18541        ];
18542
18543        for (options, expected_kind) in cases {
18544            let ctx = workflow_context(Vec::new());
18545            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18546            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18547            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18548                call.as_mut().poll(&mut task_context)
18549            else {
18550                panic!("expected typed activity validation error");
18551            };
18552            assert_eq!(error.kind, expected_kind);
18553            assert!(ctx.take_commands().expect("commands").is_empty());
18554        }
18555    }
18556
18557    #[test]
18558    fn activity_options_validate_positive_and_ordered_timeouts() {
18559        let zero_timeout_cases = [
18560            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18561            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18562            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18563            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18564        ];
18565        for options in zero_timeout_cases {
18566            assert_eq!(
18567                options.validate().expect_err("zero timeout").kind,
18568                ActivityOptionsErrorKind::TimeoutNotPositive
18569            );
18570        }
18571
18572        let ordering_cases = [
18573            ActivityOptions::new()
18574                .heartbeat_timeout(Duration::from_secs(11))
18575                .start_to_close_timeout(Duration::from_secs(10)),
18576            ActivityOptions::new()
18577                .start_to_close_timeout(Duration::from_secs(31))
18578                .schedule_to_close_timeout(Duration::from_secs(30)),
18579            ActivityOptions::new()
18580                .schedule_to_start_timeout(Duration::from_secs(31))
18581                .schedule_to_close_timeout(Duration::from_secs(30)),
18582        ];
18583        for options in ordering_cases {
18584            assert_eq!(
18585                options.validate().expect_err("timeout order").kind,
18586                ActivityOptionsErrorKind::TimeoutOrder
18587            );
18588        }
18589
18590        assert_eq!(
18591            ActivityOptions::new()
18592                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18593                .validate()
18594                .expect_err("protocol integer overflow")
18595                .kind,
18596            ActivityOptionsErrorKind::TimeoutOverflow
18597        );
18598    }
18599
18600    #[test]
18601    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18602        let ctx = workflow_context(completed_retry_activity_history());
18603        let mut call =
18604            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18605        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18606
18607        assert!(matches!(
18608            call.as_mut().poll(&mut task_context),
18609            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18610        ));
18611        assert!(ctx.take_commands().expect("commands").is_empty());
18612        ctx.ensure_history_consumed().expect("history consumed");
18613    }
18614
18615    #[test]
18616    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18617        let mut options = retry_activity_options();
18618        options
18619            .retry_policy
18620            .as_mut()
18621            .expect("retry policy")
18622            .non_retryable_error_types
18623            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18624
18625        let new_ctx = workflow_context(Vec::new());
18626        let mut new_call =
18627            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18628        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18629        assert!(matches!(
18630            new_call.as_mut().poll(&mut task_context),
18631            Poll::Pending
18632        ));
18633        let commands = new_ctx.take_commands().expect("commands");
18634        assert_eq!(commands.len(), 1);
18635        assert_eq!(
18636            commands[0]["retry_policy"]["non_retryable_error_types"],
18637            json!(["PermanentError"])
18638        );
18639
18640        let replay_ctx = workflow_context(completed_retry_activity_history());
18641        let mut replay_call =
18642            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18643        assert!(matches!(
18644            replay_call.as_mut().poll(&mut task_context),
18645            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18646        ));
18647        assert!(replay_ctx.take_commands().expect("commands").is_empty());
18648        replay_ctx
18649            .ensure_history_consumed()
18650            .expect("history consumed");
18651    }
18652
18653    #[test]
18654    fn replayed_intermediate_retry_remains_pending_across_restarts() {
18655        let history = completed_retry_activity_history()
18656            .into_iter()
18657            .take(3)
18658            .collect::<Vec<_>>();
18659
18660        for _restart in 0..2 {
18661            let ctx = workflow_context(history.clone());
18662            let mut call =
18663                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18664            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18665            assert!(matches!(
18666                call.as_mut().poll(&mut task_context),
18667                Poll::Pending
18668            ));
18669            assert!(ctx.take_commands().expect("commands").is_empty());
18670        }
18671    }
18672
18673    #[test]
18674    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18675        let mut changed_queue = retry_activity_options();
18676        changed_queue.task_queue = Some("different-queue".to_string());
18677
18678        let mut changed_max_attempts = retry_activity_options();
18679        let retry_policy = changed_max_attempts
18680            .retry_policy
18681            .as_mut()
18682            .expect("retry policy");
18683        retry_policy.max_attempts = Some(4);
18684
18685        let mut changed_backoff = retry_activity_options();
18686        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18687        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18688            Duration::from_secs(3),
18689            Duration::from_secs(4),
18690        ]));
18691
18692        let mut changed_non_retryable_types = retry_activity_options();
18693        let retry_policy = changed_non_retryable_types
18694            .retry_policy
18695            .as_mut()
18696            .expect("retry policy");
18697        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18698
18699        let mut changed_start_to_close = retry_activity_options();
18700        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18701        let mut changed_schedule_to_start = retry_activity_options();
18702        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18703        let mut changed_schedule_to_close = retry_activity_options();
18704        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18705        let mut changed_heartbeat = retry_activity_options();
18706        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18707
18708        let cases = [
18709            (changed_queue, "activity_task_queue_mismatch"),
18710            (changed_max_attempts, "activity_retry_policy_mismatch"),
18711            (changed_backoff, "activity_retry_policy_mismatch"),
18712            (
18713                changed_non_retryable_types,
18714                "activity_retry_policy_mismatch",
18715            ),
18716            (changed_start_to_close, "activity_retry_policy_mismatch"),
18717            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18718            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18719            (changed_heartbeat, "activity_retry_policy_mismatch"),
18720        ];
18721
18722        for (options, expected_reason) in cases {
18723            let ctx = workflow_context(completed_retry_activity_history());
18724            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18725            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18726            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18727                call.as_mut().poll(&mut task_context)
18728            else {
18729                panic!("changed activity options must fail replay");
18730            };
18731            assert_eq!(failure.reason, expected_reason);
18732            assert_eq!(failure.sequence, Some(1));
18733            assert!(ctx.take_commands().expect("commands").is_empty());
18734        }
18735    }
18736
18737    #[test]
18738    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18739        let cases = [
18740            (
18741                "execution_mode",
18742                json!("local"),
18743                "activity_execution_mode_mismatch",
18744            ),
18745            (
18746                "snapshot_version",
18747                json!(2),
18748                "activity_retry_policy_mismatch",
18749            ),
18750        ];
18751
18752        for (field, value, expected_reason) in cases {
18753            let mut history = completed_retry_activity_history();
18754            let activity = history[0].payload["activity"]
18755                .as_object_mut()
18756                .expect("activity snapshot");
18757            if field == "execution_mode" {
18758                activity.insert(field.to_string(), value);
18759            } else {
18760                activity["retry_policy"]
18761                    .as_object_mut()
18762                    .expect("retry snapshot")
18763                    .insert(field.to_string(), value);
18764            }
18765
18766            let ctx = workflow_context(history);
18767            let mut call =
18768                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18769            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18770            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18771                call.as_mut().poll(&mut task_context)
18772            else {
18773                panic!("changed {field} must fail replay");
18774            };
18775            assert_eq!(failure.reason, expected_reason);
18776            assert_eq!(failure.sequence, Some(1));
18777            assert!(ctx.take_commands().expect("commands").is_empty());
18778        }
18779    }
18780
18781    #[test]
18782    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18783        let mut history = completed_retry_activity_history();
18784        let activity = history[0].payload["activity"]
18785            .as_object_mut()
18786            .expect("activity snapshot");
18787        activity.remove("execution_mode");
18788        activity.remove("retry_policy");
18789
18790        let mut current = retry_activity_options();
18791        current.start_to_close_timeout = Some(Duration::from_secs(45));
18792        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18793        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18794        current.heartbeat_timeout = Some(Duration::from_secs(12));
18795
18796        let ctx = workflow_context(history);
18797        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18798        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18799        assert!(matches!(
18800            call.as_mut().poll(&mut task_context),
18801            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18802        ));
18803        assert!(ctx.take_commands().expect("commands").is_empty());
18804        ctx.ensure_history_consumed().expect("history consumed");
18805    }
18806
18807    #[test]
18808    fn terminal_activity_failed_after_start_returns_typed_failure() {
18809        let history = vec![
18810            history_event(
18811                "ActivityScheduled",
18812                json!({
18813                    "sequence": 1,
18814                    "activity_type": "flaky",
18815                    "activity_execution_id": "act-terminal",
18816                    "activity": {
18817                        "id": "act-terminal",
18818                        "sequence": 1,
18819                        "type": "flaky",
18820                        "queue": "critical-activities",
18821                        "retry_policy": {
18822                            "snapshot_version": 1,
18823                            "max_attempts": 3,
18824                            "backoff_seconds": [2, 4],
18825                            "non_retryable_error_types": ["PermanentError"]
18826                        }
18827                    }
18828                }),
18829            ),
18830            history_event(
18831                "ActivityStarted",
18832                json!({
18833                    "sequence": 1,
18834                    "activity_type": "flaky",
18835                    "activity_execution_id": "act-terminal",
18836                    "activity_attempt_id": "attempt-1",
18837                    "attempt_number": 1
18838                }),
18839            ),
18840            history_event(
18841                "ActivityFailed",
18842                json!({
18843                    "sequence": 1,
18844                    "activity_type": "flaky",
18845                    "activity_execution_id": "act-terminal",
18846                    "activity_attempt_id": "attempt-1",
18847                    "attempt_number": 1,
18848                    "failure_id": "failure-terminal",
18849                    "failure_category": "activity",
18850                    "exception_type": "PermanentError",
18851                    "message": "cannot retry",
18852                    "non_retryable": true
18853                }),
18854            ),
18855        ];
18856        let ctx = workflow_context(history);
18857        let mut call =
18858            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18859        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18860
18861        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18862            call.as_mut().poll(&mut task_context)
18863        else {
18864            panic!("terminal ActivityFailed must settle the activity future");
18865        };
18866        assert_eq!(failure.kind, ActivityFailureKind::Failed);
18867        assert_eq!(
18868            failure.activity_execution_id.as_deref(),
18869            Some("act-terminal")
18870        );
18871        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18872        assert!(failure.non_retryable);
18873        assert!(ctx.take_commands().expect("commands").is_empty());
18874        ctx.ensure_history_consumed().expect("history consumed");
18875    }
18876
18877    #[test]
18878    fn activity_terminal_events_return_machine_readable_failures() {
18879        let cases = [
18880            (
18881                "ActivityFailed",
18882                json!({
18883                    "sequence": 1,
18884                    "activity_type": "charge-card",
18885                    "activity_execution_id": "act-1",
18886                    "activity_attempt_id": "attempt-2",
18887                    "attempt_number": 2,
18888                    "failure_id": "failure-1",
18889                    "failure_category": "activity",
18890                    "exception_type": "PaymentDeclined",
18891                    "exception_class": "payments.PaymentDeclined",
18892                    "message": "card declined",
18893                    "non_retryable": true
18894                }),
18895                ActivityFailureKind::Failed,
18896                "activity",
18897            ),
18898            (
18899                "ActivityCancelled",
18900                json!({
18901                    "sequence": 1,
18902                    "activity_type": "charge-card",
18903                    "activity_execution_id": "act-1",
18904                    "activity_attempt_id": "attempt-1"
18905                }),
18906                ActivityFailureKind::Cancelled,
18907                "cancelled",
18908            ),
18909        ];
18910
18911        for (event_type, payload, expected_kind, expected_reason) in cases {
18912            let ctx = workflow_context(vec![history_event(event_type, payload)]);
18913            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18914            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18915            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18916                call.as_mut().poll(&mut task_context)
18917            else {
18918                panic!("expected terminal activity failure");
18919            };
18920            assert_eq!(failure.kind, expected_kind);
18921            assert_eq!(failure.reason, expected_reason);
18922            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18923            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18924        }
18925    }
18926
18927    #[test]
18928    fn every_activity_timeout_class_is_typed() {
18929        for timeout_kind in [
18930            "start_to_close",
18931            "schedule_to_start",
18932            "schedule_to_close",
18933            "heartbeat",
18934        ] {
18935            let ctx = workflow_context(vec![history_event(
18936                "ActivityTimedOut",
18937                json!({
18938                    "sequence": 1,
18939                    "activity_type": "slow",
18940                    "activity_execution_id": "act-timeout",
18941                    "activity_attempt_id": "attempt-timeout",
18942                    "failure_category": "timeout",
18943                    "timeout_kind": timeout_kind,
18944                    "message": "deadline expired"
18945                }),
18946            )]);
18947            let mut call = Box::pin(ctx.activity("slow", json!([])));
18948            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18949            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18950                call.as_mut().poll(&mut task_context)
18951            else {
18952                panic!("expected timeout failure");
18953            };
18954            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18955            assert_eq!(failure.reason, timeout_kind);
18956            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18957            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18958        }
18959    }
18960
18961    #[test]
18962    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18963        let ctx = workflow_context(Vec::new());
18964        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18965        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18966
18967        assert!(matches!(
18968            sleep.as_mut().poll(&mut task_context),
18969            Poll::Pending
18970        ));
18971        assert!(matches!(
18972            sleep.as_mut().poll(&mut task_context),
18973            Poll::Pending
18974        ));
18975
18976        let commands = ctx.take_commands().expect("timer command");
18977        assert_eq!(
18978            commands,
18979            vec![json!({
18980                "type": "start_timer",
18981                "delay_seconds": 2,
18982            })]
18983        );
18984    }
18985
18986    #[test]
18987    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18988        let history = vec![
18989            history_event(
18990                "TimerScheduled",
18991                json!({
18992                    "sequence": 1,
18993                    "timer_id": "timer-1",
18994                    "delay_seconds": 5,
18995                    "fire_at": "2026-07-11T12:00:05Z",
18996                }),
18997            ),
18998            history_event(
18999                "TimerFired",
19000                json!({
19001                    "sequence": 1,
19002                    "timer_id": "timer-1",
19003                    "delay_seconds": 5,
19004                    "fire_at": "2026-07-11T12:00:05Z",
19005                    "fired_at": "2026-07-11T12:00:05Z",
19006                }),
19007            ),
19008        ];
19009
19010        for _restart in 0..2 {
19011            let ctx = workflow_context(history.clone());
19012            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
19013            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19014            assert!(matches!(
19015                sleep.as_mut().poll(&mut task_context),
19016                Poll::Ready(Ok(()))
19017            ));
19018            assert!(ctx.take_commands().expect("commands").is_empty());
19019            ctx.ensure_history_consumed().expect("history consumed");
19020        }
19021    }
19022
19023    #[test]
19024    fn workflow_sleep_rejects_changed_delay_during_replay() {
19025        let ctx = workflow_context(vec![
19026            history_event(
19027                "TimerScheduled",
19028                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19029            ),
19030            history_event(
19031                "TimerFired",
19032                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19033            ),
19034        ]);
19035        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
19036        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19037
19038        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19039            sleep.as_mut().poll(&mut task_context)
19040        else {
19041            panic!("changed timer delay must be rejected");
19042        };
19043        assert_eq!(failure.reason, "timer_delay_mismatch");
19044        assert_eq!(failure.sequence, Some(1));
19045    }
19046
19047    #[test]
19048    fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
19049        let ctx = workflow_context(Vec::new());
19050        let mut wait = Box::pin(
19051            ctx.wait_condition(
19052                ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
19053                    .timeout(Duration::from_millis(60_001)),
19054                || Ok(false),
19055            ),
19056        );
19057        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19058
19059        assert!(matches!(
19060            wait.as_mut().poll(&mut task_context),
19061            Poll::Pending
19062        ));
19063        assert!(matches!(
19064            wait.as_mut().poll(&mut task_context),
19065            Poll::Pending
19066        ));
19067        assert_eq!(
19068            ctx.take_commands().expect("condition command"),
19069            vec![json!({
19070                "type": "open_condition_wait",
19071                "condition_wait_occurrence_id": "rust:condition-wait:0",
19072                "condition_key": "approval.ready",
19073                "condition_definition_fingerprint": "sha256:approval-v1",
19074                "timeout_seconds": 61,
19075            })]
19076        );
19077    }
19078
19079    #[test]
19080    fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
19081        let ctx = workflow_context(Vec::new());
19082        let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
19083        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19084        assert!(matches!(
19085            satisfied.as_mut().poll(&mut task_context),
19086            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19087        ));
19088
19089        let mut timed_out = Box::pin(wait_condition!(
19090            ctx,
19091            "no-wait",
19092            timeout: Duration::ZERO,
19093            || Ok(false),
19094        ));
19095        assert!(matches!(
19096            timed_out.as_mut().poll(&mut task_context),
19097            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
19098        ));
19099        assert!(ctx.take_commands().expect("commands").is_empty());
19100    }
19101
19102    #[test]
19103    fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
19104        let signal_history = vec![
19105            history_event(
19106                "ConditionWaitOpened",
19107                json!({
19108                    "sequence": 4,
19109                    "condition_wait_id": "condition:4",
19110                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19111                    "condition_key": "approval",
19112                    "condition_definition_fingerprint": "sha256:approval-v1",
19113                    "timeout_seconds": 30,
19114                }),
19115            ),
19116            history_event(
19117                "SignalReceived",
19118                json!({
19119                    "workflow_sequence": 4,
19120                    "signal_name": "approve",
19121                    "arguments": fixture_envelope(json!(["Ada"])),
19122                }),
19123            ),
19124        ];
19125        for _worker_before_or_after_restart in 0..2 {
19126            let ctx = workflow_context(signal_history.clone());
19127            let predicate_ctx = ctx.clone();
19128            let mut wait = Box::pin(
19129                ctx.wait_condition(
19130                    ConditionWaitOptions::new("approval", "sha256:approval-v1")
19131                        .timeout(Duration::from_secs(30)),
19132                    move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
19133                ),
19134            );
19135            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19136            assert!(matches!(
19137                wait.as_mut().poll(&mut task_context),
19138                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19139            ));
19140            assert!(ctx.take_commands().expect("commands").is_empty());
19141            ctx.ensure_history_consumed().expect("condition consumed");
19142        }
19143
19144        let update_history = vec![
19145            history_event(
19146                "ConditionWaitOpened",
19147                json!({
19148                    "sequence": 7,
19149                    "condition_wait_id": "condition:7",
19150                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19151                    "condition_key": "update-approval",
19152                    "condition_definition_fingerprint": "sha256:update-approval-v1",
19153                }),
19154            ),
19155            history_event(
19156                "UpdateApplied",
19157                json!({
19158                    "sequence": 7,
19159                    "update_id": "update-1",
19160                    "update_name": "approve",
19161                    "arguments": fixture_envelope(json!([true])),
19162                }),
19163            ),
19164        ];
19165        let ctx = workflow_context(update_history);
19166        let predicate_ctx = ctx.clone();
19167        let mut wait = Box::pin(ctx.wait_condition(
19168            ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
19169            move || {
19170                Ok(predicate_ctx
19171                    .updates("approve")?
19172                    .first()
19173                    .and_then(|arguments| arguments.first())
19174                    .and_then(Value::as_bool)
19175                    == Some(true))
19176            },
19177        ));
19178        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19179        assert!(matches!(
19180            wait.as_mut().poll(&mut task_context),
19181            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19182        ));
19183        assert!(ctx.take_commands().expect("commands").is_empty());
19184        ctx.ensure_history_consumed().expect("condition consumed");
19185    }
19186
19187    #[test]
19188    fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
19189        let open_history = vec![
19190            history_event(
19191                "ConditionWaitOpened",
19192                json!({
19193                    "sequence": 3,
19194                    "condition_wait_id": "condition:3",
19195                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19196                    "condition_key": "two-votes",
19197                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19198                    "timeout_seconds": 120,
19199                }),
19200            ),
19201            history_event(
19202                "SignalReceived",
19203                json!({
19204                    "workflow_sequence": 3,
19205                    "signal_name": "vote",
19206                    "arguments": fixture_envelope(json!(["first"])),
19207                }),
19208            ),
19209        ];
19210        for _worker_before_or_after_restart in 0..2 {
19211            let ctx = workflow_context(open_history.clone());
19212            let predicate_ctx = ctx.clone();
19213            let mut wait = Box::pin(
19214                ctx.wait_condition(
19215                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
19216                        .timeout(Duration::from_secs(120)),
19217                    move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19218                ),
19219            );
19220            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19221            assert!(matches!(
19222                wait.as_mut().poll(&mut task_context),
19223                Poll::Pending
19224            ));
19225            assert_eq!(
19226                ctx.take_commands().expect("reopened condition"),
19227                vec![json!({
19228                    "type": "open_condition_wait",
19229                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19230                    "condition_key": "two-votes",
19231                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19232                    "timeout_seconds": 120,
19233                })]
19234            );
19235        }
19236
19237        let satisfied_ctx = workflow_context(vec![
19238            history_event(
19239                "ConditionWaitOpened",
19240                json!({
19241                    "sequence": 5,
19242                    "condition_wait_id": "condition:5",
19243                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19244                    "condition_key": "approval",
19245                    "condition_definition_fingerprint": "sha256:approval-v1",
19246                }),
19247            ),
19248            history_event(
19249                "ConditionWaitSatisfied",
19250                json!({
19251                    "sequence": 5,
19252                    "condition_wait_id": "condition:5",
19253                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19254                    "condition_key": "approval",
19255                    "condition_definition_fingerprint": "sha256:approval-v1",
19256                }),
19257            ),
19258        ]);
19259        let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
19260            ConditionWaitOptions::new("approval", "sha256:approval-v1"),
19261            || Ok(false),
19262        ));
19263        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19264        assert!(matches!(
19265            satisfied.as_mut().poll(&mut task_context),
19266            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19267        ));
19268
19269        let timed_out_ctx = workflow_context(vec![
19270            history_event(
19271                "ConditionWaitOpened",
19272                json!({
19273                    "sequence": 8,
19274                    "condition_wait_id": "condition:8",
19275                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19276                    "condition_key": "approval-timeout",
19277                    "condition_definition_fingerprint": "sha256:approval-timeout-v1",
19278                    "timeout_seconds": 5,
19279                }),
19280            ),
19281            history_event(
19282                "TimerScheduled",
19283                json!({
19284                    "sequence": 9,
19285                    "timer_id": "condition-timer:9",
19286                    "timer_kind": "condition_timeout",
19287                    "condition_wait_id": "condition:8",
19288                    "delay_seconds": 5,
19289                }),
19290            ),
19291            history_event(
19292                "TimerFired",
19293                json!({
19294                    "sequence": 9,
19295                    "timer_id": "condition-timer:9",
19296                    "timer_kind": "condition_timeout",
19297                    "condition_wait_id": "condition:8",
19298                    "delay_seconds": 5,
19299                }),
19300            ),
19301        ]);
19302        let mut timed_out = Box::pin(
19303            timed_out_ctx.wait_condition(
19304                ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
19305                    .timeout(Duration::from_secs(5)),
19306                || Ok(true),
19307            ),
19308        );
19309        assert!(matches!(
19310            timed_out.as_mut().poll(&mut task_context),
19311            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
19312        ));
19313    }
19314
19315    #[test]
19316    fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
19317        let history = vec![
19318            history_event(
19319                "ConditionWaitOpened",
19320                json!({
19321                    "sequence": 3,
19322                    "condition_wait_id": "condition:3",
19323                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19324                    "condition_key": "two-votes",
19325                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19326                }),
19327            ),
19328            history_event(
19329                "SignalReceived",
19330                json!({
19331                    "workflow_sequence": 3,
19332                    "signal_name": "vote",
19333                    "arguments": fixture_envelope(json!(["first"])),
19334                }),
19335            ),
19336            history_event(
19337                "ConditionWaitSatisfied",
19338                json!({
19339                    "sequence": 3,
19340                    "condition_wait_id": "condition:3",
19341                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19342                    "condition_key": "two-votes",
19343                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19344                }),
19345            ),
19346            history_event(
19347                "ConditionWaitOpened",
19348                json!({
19349                    "sequence": 5,
19350                    "condition_wait_id": "condition:5",
19351                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19352                    "condition_key": "two-votes",
19353                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19354                }),
19355            ),
19356            history_event(
19357                "SignalReceived",
19358                json!({
19359                    "workflow_sequence": 5,
19360                    "signal_name": "vote",
19361                    "arguments": fixture_envelope(json!(["second"])),
19362                }),
19363            ),
19364            history_event(
19365                "ConditionWaitSatisfied",
19366                json!({
19367                    "sequence": 5,
19368                    "condition_wait_id": "condition:5",
19369                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19370                    "condition_key": "two-votes",
19371                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19372                }),
19373            ),
19374        ];
19375        for _cold_worker_or_restart in 0..2 {
19376            let ctx = workflow_context(history.clone());
19377            let predicate_ctx = ctx.clone();
19378            let mut wait = Box::pin(ctx.wait_condition(
19379                ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19380                move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19381            ));
19382            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19383
19384            assert!(matches!(
19385                wait.as_mut().poll(&mut task_context),
19386                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19387            ));
19388            assert!(ctx.take_commands().expect("commands").is_empty());
19389            ctx.ensure_history_consumed()
19390                .expect("every physical wait-open is consumed");
19391        }
19392    }
19393
19394    #[test]
19395    fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19396        let history = vec![
19397            history_event(
19398                "ConditionWaitOpened",
19399                json!({
19400                    "sequence": 3,
19401                    "condition_wait_id": "condition:3",
19402                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19403                    "condition_key": "approved",
19404                    "condition_definition_fingerprint": "sha256:approved-v1",
19405                }),
19406            ),
19407            history_event(
19408                "UpdateApplied",
19409                json!({
19410                    "sequence": 3,
19411                    "update_id": "update-1",
19412                    "update_name": "approve",
19413                    "arguments": fixture_envelope(json!([false])),
19414                }),
19415            ),
19416            history_event(
19417                "ConditionWaitOpened",
19418                json!({
19419                    "sequence": 5,
19420                    "condition_wait_id": "condition:5",
19421                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19422                    "condition_key": "approved",
19423                    "condition_definition_fingerprint": "sha256:approved-v1",
19424                }),
19425            ),
19426            history_event(
19427                "UpdateApplied",
19428                json!({
19429                    "sequence": 5,
19430                    "update_id": "update-2",
19431                    "update_name": "approve",
19432                    "arguments": fixture_envelope(json!([true])),
19433                }),
19434            ),
19435        ];
19436
19437        for _cold_worker_or_restart in 0..2 {
19438            let ctx = workflow_context(history.clone());
19439            let predicate_ctx = ctx.clone();
19440            let mut wait = Box::pin(ctx.wait_condition(
19441                ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19442                move || {
19443                    Ok(predicate_ctx
19444                        .updates("approve")?
19445                        .last()
19446                        .and_then(|arguments| arguments.first())
19447                        .and_then(Value::as_bool)
19448                        == Some(true))
19449                },
19450            ));
19451            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19452
19453            assert!(matches!(
19454                wait.as_mut().poll(&mut task_context),
19455                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19456            ));
19457            assert!(ctx.take_commands().expect("commands").is_empty());
19458            ctx.ensure_history_consumed()
19459                .expect("every update-driven reopen is consumed");
19460        }
19461    }
19462
19463    #[test]
19464    fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19465        for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19466            ("shared", "sha256:first", "shared", "sha256:second"),
19467            ("first", "sha256:shared", "second", "sha256:shared"),
19468            ("shared", "sha256:shared", "shared", "sha256:shared"),
19469            ("first", "sha256:first", "second", "sha256:second"),
19470        ] {
19471            let history = vec![
19472                history_event(
19473                    "ConditionWaitOpened",
19474                    json!({
19475                        "sequence": 3,
19476                        "condition_wait_id": "condition:3",
19477                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19478                        "condition_key": first_key,
19479                        "condition_definition_fingerprint": first_fingerprint,
19480                    }),
19481                ),
19482                history_event(
19483                    "ConditionWaitSatisfied",
19484                    json!({
19485                        "sequence": 3,
19486                        "condition_wait_id": "condition:3",
19487                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19488                        "condition_key": first_key,
19489                        "condition_definition_fingerprint": first_fingerprint,
19490                    }),
19491                ),
19492                history_event(
19493                    "ConditionWaitOpened",
19494                    json!({
19495                        "sequence": 4,
19496                        "condition_wait_id": "condition:4",
19497                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19498                        "condition_key": second_key,
19499                        "condition_definition_fingerprint": second_fingerprint,
19500                    }),
19501                ),
19502                history_event(
19503                    "ConditionWaitSatisfied",
19504                    json!({
19505                        "sequence": 4,
19506                        "condition_wait_id": "condition:4",
19507                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19508                        "condition_key": second_key,
19509                        "condition_definition_fingerprint": second_fingerprint,
19510                    }),
19511                ),
19512            ];
19513            for _cold_worker_or_restart in 0..2 {
19514                let ctx = workflow_context(history.clone());
19515                let mut task_context = TaskContext::from_waker(noop_waker_ref());
19516                let mut first = Box::pin(ctx.wait_condition(
19517                    ConditionWaitOptions::new(first_key, first_fingerprint),
19518                    || Ok(false),
19519                ));
19520                assert!(matches!(
19521                    first.as_mut().poll(&mut task_context),
19522                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19523                ));
19524
19525                let mut second = Box::pin(ctx.wait_condition(
19526                    ConditionWaitOptions::new(second_key, second_fingerprint),
19527                    || Ok(false),
19528                ));
19529                assert!(matches!(
19530                    second.as_mut().poll(&mut task_context),
19531                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19532                ));
19533                assert!(ctx.take_commands().expect("commands").is_empty());
19534                ctx.ensure_history_consumed()
19535                    .expect("each authored wait consumes one occurrence");
19536            }
19537        }
19538    }
19539
19540    #[test]
19541    fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19542        fn worker() -> Worker {
19543            let client = Client::new("http://127.0.0.1:8080").expect("client");
19544            let mut worker = Worker::new(client, "rust-workers");
19545            worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19546                let mut outcomes = Vec::new();
19547                for _ in 0..2 {
19548                    outcomes.push(
19549                        ctx.wait_condition(
19550                            ConditionWaitOptions::new("shared", "sha256:shared"),
19551                            || Ok(false),
19552                        )
19553                        .await?,
19554                    );
19555                }
19556                Ok(json!(outcomes))
19557            });
19558            worker
19559        }
19560
19561        let task = workflow_task(
19562            "rust.condition-loop",
19563            vec![
19564                history_event(
19565                    "ConditionWaitOpened",
19566                    json!({
19567                        "sequence": 1,
19568                        "condition_wait_id": "condition:1",
19569                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19570                        "condition_key": "shared",
19571                        "condition_definition_fingerprint": "sha256:shared",
19572                    }),
19573                ),
19574                history_event(
19575                    "ConditionWaitSatisfied",
19576                    json!({
19577                        "sequence": 1,
19578                        "condition_wait_id": "condition:1",
19579                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19580                        "condition_key": "shared",
19581                        "condition_definition_fingerprint": "sha256:shared",
19582                    }),
19583                ),
19584                history_event(
19585                    "ConditionWaitOpened",
19586                    json!({
19587                        "sequence": 2,
19588                        "condition_wait_id": "condition:2",
19589                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19590                        "condition_key": "shared",
19591                        "condition_definition_fingerprint": "sha256:shared",
19592                    }),
19593                ),
19594                history_event(
19595                    "ConditionWaitSatisfied",
19596                    json!({
19597                        "sequence": 2,
19598                        "condition_wait_id": "condition:2",
19599                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19600                        "condition_key": "shared",
19601                        "condition_definition_fingerprint": "sha256:shared",
19602                    }),
19603                ),
19604            ],
19605            DEFAULT_CODEC,
19606        );
19607
19608        for _cold_worker_or_restart in 0..2 {
19609            let commands = worker()
19610                .execute_workflow_task(task.clone())
19611                .expect("adjacent loop waits replay deterministically");
19612            assert_eq!(commands.len(), 1);
19613            assert_eq!(commands[0]["type"], "complete_workflow");
19614            assert_eq!(
19615                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19616                json!(["satisfied", "satisfied"])
19617            );
19618        }
19619    }
19620
19621    #[test]
19622    fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19623        let history = vec![history_event(
19624            "ConditionWaitOpened",
19625            json!({
19626                "sequence": 12,
19627                "condition_wait_id": "condition:12",
19628                "condition_wait_occurrence_id": "rust:condition-wait:0",
19629                "condition_key": "approval",
19630                "condition_definition_fingerprint": "sha256:approval-v1",
19631                "timeout_seconds": 30,
19632            }),
19633        )];
19634        for (options, expected_reason) in [
19635            (
19636                ConditionWaitOptions::new("changed", "sha256:approval-v1")
19637                    .timeout(Duration::from_secs(30)),
19638                "condition_wait_key_mismatch",
19639            ),
19640            (
19641                ConditionWaitOptions::new("approval", "sha256:approval-v2")
19642                    .timeout(Duration::from_secs(30)),
19643                "condition_wait_predicate_mismatch",
19644            ),
19645            (
19646                ConditionWaitOptions::new("approval", "sha256:approval-v1")
19647                    .timeout(Duration::from_secs(29)),
19648                "condition_wait_timeout_mismatch",
19649            ),
19650        ] {
19651            let ctx = workflow_context(history.clone());
19652            let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19653            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19654            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19655                wait.as_mut().poll(&mut task_context)
19656            else {
19657                panic!("changed condition definition must fail replay");
19658            };
19659            assert_eq!(failure.reason, expected_reason);
19660            assert_eq!(failure.sequence, Some(12));
19661        }
19662    }
19663
19664    #[test]
19665    fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19666        let error = WorkflowState::new(
19667            vec![history_event(
19668                "ConditionWaitOpened",
19669                json!({
19670                    "sequence": 12,
19671                    "condition_wait_id": "condition:12",
19672                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19673                    "condition_key": "approval",
19674                }),
19675            )],
19676            "rust-workers".to_string(),
19677            DEFAULT_CODEC.to_string(),
19678            None,
19679        )
19680        .expect_err("condition history without a predicate fingerprint must fail");
19681
19682        assert!(matches!(
19683            error,
19684            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19685                if reason == "condition_wait_predicate_fingerprint_missing"
19686        ));
19687    }
19688
19689    #[test]
19690    fn condition_wait_history_requires_authored_occurrence_identity() {
19691        let error = WorkflowState::new(
19692            vec![history_event(
19693                "ConditionWaitOpened",
19694                json!({
19695                    "sequence": 12,
19696                    "condition_wait_id": "condition:12",
19697                    "condition_key": "approval",
19698                    "condition_definition_fingerprint": "sha256:approval-v1",
19699                }),
19700            )],
19701            "rust-workers".to_string(),
19702            DEFAULT_CODEC.to_string(),
19703            None,
19704        )
19705        .expect_err("condition history without occurrence identity must fail");
19706
19707        assert!(matches!(
19708            error,
19709            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19710                if reason == "condition_wait_occurrence_id_missing"
19711        ));
19712    }
19713
19714    #[test]
19715    fn typed_search_attribute_updates_validate_emit_and_replay() {
19716        let update = SearchAttributeUpdate::new()
19717            .keyword("OrderStatus", " waiting ")
19718            .expect("keyword")
19719            .int("Attempt", 3)
19720            .expect("int")
19721            .bool("Escalated", false)
19722            .expect("bool")
19723            .keyword_list("Regions", ["us-east", "eu-west"])
19724            .expect("list")
19725            .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19726            .expect("datetime")
19727            .delete("LegacyStatus")
19728            .expect("delete");
19729        let ctx = workflow_context(Vec::new());
19730        ctx.upsert_search_attributes(update.clone())
19731            .expect("typed update");
19732        assert_eq!(
19733            ctx.take_commands().expect("search-attribute command"),
19734            vec![json!({
19735                "type": "upsert_search_attributes",
19736                "attributes": {
19737                    "Attempt": 3,
19738                    "Escalated": false,
19739                    "LegacyStatus": null,
19740                    "OrderStatus": "waiting",
19741                    "Regions": ["us-east", "eu-west"],
19742                    "UpdatedAt": "2026-08-22T04:00:00Z",
19743                },
19744                "attribute_types": {
19745                    "Attempt": "int",
19746                    "Escalated": "bool",
19747                    "OrderStatus": "keyword",
19748                    "Regions": "keyword_list",
19749                    "UpdatedAt": "datetime",
19750                },
19751            })]
19752        );
19753
19754        let replay = workflow_context(vec![history_event(
19755            "SearchAttributesUpserted",
19756            json!({
19757                "sequence": 6,
19758                "attributes": {
19759                    "Attempt": 3,
19760                    "Escalated": false,
19761                    "LegacyStatus": null,
19762                    "OrderStatus": "waiting",
19763                    "Regions": ["us-east", "eu-west"],
19764                    "UpdatedAt": "2026-08-22T04:00:00Z",
19765                },
19766                "attribute_types": {
19767                    "Attempt": "int",
19768                    "Escalated": "bool",
19769                    "OrderStatus": "keyword",
19770                    "Regions": "keyword_list",
19771                    "UpdatedAt": "datetime",
19772                },
19773                "merged": {},
19774            }),
19775        )]);
19776        replay
19777            .upsert_search_attributes(update)
19778            .expect("matching update replays");
19779        assert!(replay.take_commands().expect("commands").is_empty());
19780        replay.ensure_history_consumed().expect("history consumed");
19781
19782        let type_drift = workflow_context(vec![history_event(
19783            "SearchAttributesUpserted",
19784            json!({
19785                "sequence": 7,
19786                "attributes": {"OrderStatus": "waiting"},
19787                "attribute_types": {"OrderStatus": "keyword"},
19788                "merged": {"OrderStatus": "waiting"},
19789            }),
19790        )]);
19791        let error = type_drift
19792            .upsert_search_attributes(
19793                SearchAttributeUpdate::new()
19794                    .string("OrderStatus", "waiting")
19795                    .expect("string update"),
19796            )
19797            .expect_err("same JSON value with a changed type must fail replay");
19798        assert!(matches!(
19799            error,
19800            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19801                if reason == "search_attribute_type_mismatch"
19802        ));
19803
19804        let malformed_types = WorkflowState::new(
19805            vec![history_event(
19806                "SearchAttributesUpserted",
19807                json!({
19808                    "sequence": 8,
19809                    "attributes": {"OrderStatus": "waiting"},
19810                    "attribute_types": {"OrderStatus": "unsupported"},
19811                    "merged": {"OrderStatus": "waiting"},
19812                }),
19813            )],
19814            "rust-workers".to_string(),
19815            DEFAULT_CODEC.to_string(),
19816            None,
19817        )
19818        .expect_err("unsupported search-attribute type metadata must fail");
19819        assert!(matches!(
19820            malformed_types,
19821            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19822                if reason == "search_attribute_types_malformed"
19823        ));
19824
19825        assert!(matches!(
19826            SearchAttributeUpdate::new().keyword("bad key", "value"),
19827            Err(SearchAttributeUpdateError::InvalidKey(_))
19828        ));
19829        assert!(matches!(
19830            SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19831            Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19832        ));
19833        assert!(matches!(
19834            SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19835            Err(SearchAttributeUpdateError::ValueTooLong { .. })
19836        ));
19837        assert!(matches!(
19838            SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19839            Err(SearchAttributeUpdateError::InvalidDateTime(_))
19840        ));
19841        assert!(matches!(
19842            workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19843            Err(Error::InvalidSearchAttributeUpdate(
19844                SearchAttributeUpdateError::Empty
19845            ))
19846        ));
19847    }
19848
19849    #[test]
19850    fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19851        let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19852        let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19853
19854        assert!(SearchAttributeUpdate::new()
19855            .string("AsciiDescription", ascii)
19856            .is_ok());
19857        assert!(SearchAttributeUpdate::new()
19858            .string("Utf8Description", utf8)
19859            .is_ok());
19860        assert!(matches!(
19861            SearchAttributeUpdate::new().string(
19862                "TooLongDescription",
19863                "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19864            ),
19865            Err(SearchAttributeUpdateError::ValueTooLong {
19866                kind: "string",
19867                limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19868                ..
19869            })
19870        ));
19871    }
19872
19873    #[test]
19874    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19875        let lone_fire = WorkflowState::new(
19876            vec![history_event(
19877                "TimerFired",
19878                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19879            )],
19880            "rust-workers".to_string(),
19881            DEFAULT_CODEC.to_string(),
19882            None,
19883        )
19884        .expect_err("TimerFired requires TimerScheduled");
19885        assert!(matches!(
19886            lone_fire,
19887            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19888                if reason == "timer_schedule_missing_or_duplicate"
19889        ));
19890
19891        let wrong_identity = WorkflowState::new(
19892            vec![
19893                history_event(
19894                    "TimerScheduled",
19895                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19896                ),
19897                history_event(
19898                    "TimerFired",
19899                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19900                ),
19901            ],
19902            "rust-workers".to_string(),
19903            DEFAULT_CODEC.to_string(),
19904            None,
19905        )
19906        .expect_err("fire must match scheduled timer identity");
19907        assert!(matches!(
19908            wrong_identity,
19909            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19910                if reason == "timer_identity_mismatch"
19911        ));
19912
19913        let duplicate_fire = WorkflowState::new(
19914            vec![
19915                history_event(
19916                    "TimerScheduled",
19917                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19918                ),
19919                history_event(
19920                    "TimerFired",
19921                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19922                ),
19923                history_event(
19924                    "TimerFired",
19925                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19926                ),
19927            ],
19928            "rust-workers".to_string(),
19929            DEFAULT_CODEC.to_string(),
19930            None,
19931        )
19932        .expect_err("a durable timer cannot fire twice");
19933        assert!(matches!(
19934            duplicate_fire,
19935            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19936                if reason == "duplicate_timer_fire"
19937        ));
19938
19939        let wrong_fired_delay = WorkflowState::new(
19940            vec![
19941                history_event(
19942                    "TimerScheduled",
19943                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19944                ),
19945                history_event(
19946                    "TimerFired",
19947                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19948                ),
19949            ],
19950            "rust-workers".to_string(),
19951            DEFAULT_CODEC.to_string(),
19952            None,
19953        )
19954        .expect_err("timer schedule and fire delays must agree");
19955        assert!(matches!(
19956            wrong_fired_delay,
19957            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19958                if reason == "timer_history_delay_mismatch"
19959        ));
19960    }
19961
19962    #[test]
19963    fn replay_rejects_activity_moved_before_recorded_timer() {
19964        let ctx = workflow_context(vec![
19965            history_event(
19966                "TimerScheduled",
19967                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19968            ),
19969            history_event(
19970                "TimerFired",
19971                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19972            ),
19973            history_event(
19974                "ActivityCompleted",
19975                json!({
19976                    "sequence": 2,
19977                    "activity_type": "after-timer",
19978                    "payload_codec": DEFAULT_CODEC,
19979                    "result": fixture_envelope(json!("done")),
19980                }),
19981            ),
19982        ]);
19983        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19984        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19985
19986        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19987            activity.as_mut().poll(&mut task_context)
19988        else {
19989            panic!("reordered durable command must be rejected");
19990        };
19991        assert_eq!(failure.reason, "recorded_command_mismatch");
19992        assert_eq!(failure.sequence, Some(1));
19993        assert_eq!(failure.expected.as_deref(), Some("timer"));
19994        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19995    }
19996
19997    #[test]
19998    fn workflow_context_emits_a_typed_named_signal_wait() {
19999        let ctx = workflow_context(Vec::new());
20000        let mut signal = Box::pin(ctx.wait_signal("finish"));
20001        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20002
20003        assert!(matches!(
20004            signal.as_mut().poll(&mut task_context),
20005            Poll::Pending
20006        ));
20007        assert_eq!(
20008            ctx.take_commands().expect("signal-wait command"),
20009            vec![json!({
20010                "type": "open_signal_wait",
20011                "signal_name": "finish",
20012            })]
20013        );
20014    }
20015
20016    #[test]
20017    fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
20018        let ctx = workflow_context(Vec::new());
20019        let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
20020        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20021
20022        let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
20023        else {
20024            panic!("runtime-reserved signal should be rejected");
20025        };
20026        assert!(message.contains("reserved by the workflow runtime"));
20027        assert!(ctx.take_commands().expect("commands").is_empty());
20028    }
20029
20030    #[tokio::test]
20031    async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
20032        let client = Client::builder("http://127.0.0.1:9")
20033            .build()
20034            .expect("client");
20035        let error = client
20036            .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
20037            .await
20038            .expect_err("runtime-reserved signal should be rejected before transport");
20039
20040        assert!(
20041            matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
20042        );
20043    }
20044
20045    #[test]
20046    fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
20047        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20048            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20049                value.to_string(),
20050            )]))
20051            .expect("message payload");
20052            json!({
20053                "schema": MESSAGE_STREAM_SCHEMA,
20054                "stream_name": "orders",
20055                "message_id": message_id,
20056                "position": position,
20057                "payload_envelope": payload,
20058            })
20059        }
20060
20061        fn opened(sequence: u64) -> HistoryEvent {
20062            history_event(
20063                "SignalWaitOpened",
20064                json!({
20065                    "sequence": sequence,
20066                    "signal_name": MESSAGE_STREAM_SIGNAL,
20067                }),
20068            )
20069        }
20070
20071        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20072            history_event(
20073                "SignalApplied",
20074                json!({
20075                    "sequence": sequence,
20076                    "signal_name": MESSAGE_STREAM_SIGNAL,
20077                    "value": fixture_envelope(json!([delivery])),
20078                }),
20079            )
20080        }
20081
20082        fn received(delivery: Value) -> HistoryEvent {
20083            history_event(
20084                "SignalReceived",
20085                json!({
20086                    "signal_name": MESSAGE_STREAM_SIGNAL,
20087                    "arguments": fixture_envelope(json!([delivery])),
20088                    "payload_codec": DEFAULT_CODEC,
20089                }),
20090            )
20091        }
20092
20093        let client = Client::new("http://127.0.0.1:8080").expect("client");
20094        let mut worker = Worker::new(client, "rust-workers");
20095        worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
20096            let messages = ctx.message_stream("orders")?.receive(2).await?;
20097            Ok(json!(messages
20098                .into_iter()
20099                .map(|message| message.message_id)
20100                .collect::<Vec<_>>()))
20101        });
20102
20103        let first = delivery("message-1", 1, "one");
20104        let second = delivery("message-2", 2, "two");
20105        let batch = worker
20106            .execute_workflow_task_decision(workflow_task(
20107                "rust.message-stream-batch",
20108                vec![
20109                    opened(1),
20110                    received(first.clone()),
20111                    applied(1, first.clone()),
20112                    received(first.clone()),
20113                    received(second),
20114                ],
20115                DEFAULT_CODEC,
20116            ))
20117            .expect("worker task consumes the available batch");
20118
20119        assert_eq!(batch.commands.len(), 1);
20120        assert_eq!(batch.commands[0]["type"], "complete_workflow");
20121        assert_eq!(
20122            decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
20123                .expect("workflow result"),
20124            json!(["message-1", "message-2"])
20125        );
20126        assert_eq!(
20127            batch.message_stream_cursors,
20128            vec![json!({"stream_name": "orders", "through_position": 2})]
20129        );
20130        assert!(batch.message_stream_waits.is_empty());
20131
20132        let partial = worker
20133            .execute_workflow_task_decision(workflow_task(
20134                "rust.message-stream-batch",
20135                vec![opened(1), received(first.clone()), applied(1, first)],
20136                DEFAULT_CODEC,
20137            ))
20138            .expect("worker task returns without waiting for a missing second item");
20139        assert_eq!(partial.commands.len(), 1);
20140        assert_eq!(partial.commands[0]["type"], "complete_workflow");
20141        assert_eq!(
20142            decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
20143                .expect("workflow result"),
20144            json!(["message-1"])
20145        );
20146        assert_eq!(
20147            partial.message_stream_cursors,
20148            vec![json!({"stream_name": "orders", "through_position": 1})]
20149        );
20150        assert!(partial.message_stream_waits.is_empty());
20151    }
20152
20153    #[test]
20154    fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
20155        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20156            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20157                value.to_string(),
20158            )]))
20159            .expect("message payload");
20160            json!({
20161                "schema": MESSAGE_STREAM_SCHEMA,
20162                "stream_name": "orders",
20163                "message_id": message_id,
20164                "position": position,
20165                "payload_envelope": payload,
20166            })
20167        }
20168
20169        fn opened(sequence: u64) -> HistoryEvent {
20170            history_event(
20171                "SignalWaitOpened",
20172                json!({
20173                    "sequence": sequence,
20174                    "signal_name": MESSAGE_STREAM_SIGNAL,
20175                }),
20176            )
20177        }
20178
20179        fn received(delivery: Value) -> HistoryEvent {
20180            history_event(
20181                "SignalReceived",
20182                json!({
20183                    "signal_name": MESSAGE_STREAM_SIGNAL,
20184                    "arguments": fixture_envelope(json!([delivery])),
20185                    "payload_codec": DEFAULT_CODEC,
20186                }),
20187            )
20188        }
20189
20190        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20191            history_event(
20192                "SignalApplied",
20193                json!({
20194                    "sequence": sequence,
20195                    "signal_name": MESSAGE_STREAM_SIGNAL,
20196                    "value": fixture_envelope(json!([delivery])),
20197                }),
20198            )
20199        }
20200
20201        let client = Client::new("http://127.0.0.1:8080").expect("client");
20202        let mut worker = Worker::new(client, "rust-workers");
20203        worker.register_workflow(
20204            "rust.message-stream-partial-batches",
20205            |ctx, _input| async move {
20206                let stream = ctx.message_stream("orders")?;
20207                let first = stream.receive(10).await?;
20208                let second = stream.receive(10).await?;
20209                Ok(json!([
20210                    first
20211                        .into_iter()
20212                        .map(|message| message.message_id)
20213                        .collect::<Vec<_>>(),
20214                    second
20215                        .into_iter()
20216                        .map(|message| message.message_id)
20217                        .collect::<Vec<_>>(),
20218                ]))
20219            },
20220        );
20221
20222        let first = delivery("message-1", 1, "one");
20223        let second = delivery("message-2", 2, "two");
20224        let decision = worker
20225            .execute_workflow_task_decision(workflow_task(
20226                "rust.message-stream-partial-batches",
20227                vec![
20228                    opened(1),
20229                    received(first.clone()),
20230                    applied(1, first),
20231                    opened(2),
20232                    received(second.clone()),
20233                    applied(2, second),
20234                ],
20235                DEFAULT_CODEC,
20236            ))
20237            .expect("cold replay preserves both authored receive boundaries");
20238
20239        assert_eq!(decision.commands.len(), 1);
20240        assert_eq!(decision.commands[0]["type"], "complete_workflow");
20241        assert_eq!(
20242            decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
20243                .expect("workflow result"),
20244            json!([["message-1"], ["message-2"]])
20245        );
20246        assert_eq!(
20247            decision.message_stream_cursors,
20248            vec![json!({"stream_name": "orders", "through_position": 2})]
20249        );
20250        assert!(decision.message_stream_waits.is_empty());
20251    }
20252
20253    #[test]
20254    fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
20255        let ctx = workflow_context(Vec::new());
20256        let stream = ctx.message_stream("orders").expect("message stream");
20257        let mut receive = Box::pin(stream.receive(10));
20258        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20259
20260        assert!(matches!(
20261            receive.as_mut().poll(&mut task_context),
20262            Poll::Pending
20263        ));
20264        assert_eq!(
20265            ctx.take_commands().expect("message-stream wait command"),
20266            vec![json!({
20267                "type": "open_signal_wait",
20268                "signal_name": MESSAGE_STREAM_SIGNAL,
20269            })]
20270        );
20271        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20272        assert!(cursors.is_empty());
20273        assert_eq!(
20274            waits,
20275            vec![json!({"stream_name": "orders", "after_position": 0})]
20276        );
20277    }
20278
20279    #[test]
20280    fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
20281        let ctx = workflow_context(vec![history_event(
20282            "SignalReceived",
20283            json!({
20284                "signal_name": MESSAGE_STREAM_SIGNAL,
20285                "arguments": fixture_envelope(json!([{
20286                    "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
20287                    "stream_name": "orders",
20288                    "through_position": 2,
20289                }])),
20290                "payload_codec": DEFAULT_CODEC,
20291            }),
20292        )]);
20293        let stream = ctx.message_stream("orders").expect("message stream");
20294        let mut receive = Box::pin(stream.receive(10));
20295        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20296
20297        assert!(matches!(
20298            receive.as_mut().poll(&mut task_context),
20299            Poll::Pending
20300        ));
20301        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20302        assert_eq!(
20303            cursors,
20304            vec![json!({"stream_name": "orders", "through_position": 2})]
20305        );
20306        assert_eq!(
20307            waits,
20308            vec![json!({"stream_name": "orders", "after_position": 2})]
20309        );
20310    }
20311
20312    #[test]
20313    fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
20314        let mut empty_map = BTreeMap::new();
20315        let mut nested = BTreeMap::new();
20316        nested.insert(
20317            "value".to_string(),
20318            AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
20319        );
20320        let values = vec![
20321            AvroValue::Bytes(vec![0, 255]),
20322            AvroValue::Long(1),
20323            AvroValue::Double(1.0),
20324            AvroValue::Array(Vec::new()),
20325            AvroValue::Map(std::mem::take(&mut empty_map)),
20326            AvroValue::Map(nested),
20327        ];
20328        let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
20329        let transport = vec![json!({
20330            "schema": MESSAGE_STREAM_SCHEMA,
20331            "stream_name": "orders",
20332            "message_id": "message-1",
20333            "position": 1,
20334            "payload_envelope": payload,
20335        })];
20336
20337        for _ in 0..2 {
20338            let Some(MessageStreamDelivery::Message(message)) =
20339                decode_message_stream_delivery(transport.clone()).expect("delivery")
20340            else {
20341                panic!("message delivery expected");
20342            };
20343            assert_eq!(message.arguments, values);
20344            assert!(matches!(message.arguments[1], AvroValue::Long(1)));
20345            assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
20346        }
20347    }
20348
20349    #[test]
20350    fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
20351        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20352            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20353                value.to_string(),
20354            )]))
20355            .expect("message payload");
20356            json!({
20357                "schema": MESSAGE_STREAM_SCHEMA,
20358                "stream_name": "orders",
20359                "message_id": message_id,
20360                "position": position,
20361                "payload_envelope": payload,
20362            })
20363        }
20364
20365        fn opened(sequence: u64) -> HistoryEvent {
20366            history_event(
20367                "SignalWaitOpened",
20368                json!({
20369                    "sequence": sequence,
20370                    "signal_name": MESSAGE_STREAM_SIGNAL,
20371                }),
20372            )
20373        }
20374
20375        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20376            history_event(
20377                "SignalApplied",
20378                json!({
20379                    "sequence": sequence,
20380                    "signal_name": MESSAGE_STREAM_SIGNAL,
20381                    "value": fixture_envelope(json!([delivery])),
20382                }),
20383            )
20384        }
20385
20386        fn worker() -> Worker {
20387            let client = Client::new("http://127.0.0.1:8080").expect("client");
20388            let mut worker = Worker::new(client, "rust-workers");
20389            worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20390                let stream = ctx.message_stream("orders")?;
20391                let first = stream.receive_one().await?;
20392                let second = stream.receive_one().await?;
20393                Ok(json!([first.message_id, second.message_id]))
20394            });
20395            worker
20396        }
20397
20398        fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20399            let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20400            task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20401            task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20402            task
20403        }
20404
20405        let waiting = worker()
20406            .execute_workflow_task_decision(workflow_task(
20407                "rust.message-stream",
20408                Vec::new(),
20409                DEFAULT_CODEC,
20410            ))
20411            .expect("first worker opens the stream wait");
20412        assert_eq!(
20413            waiting.commands,
20414            vec![json!({
20415                "type": "open_signal_wait",
20416                "signal_name": MESSAGE_STREAM_SIGNAL,
20417            })]
20418        );
20419        assert!(waiting.message_stream_cursors.is_empty());
20420        assert_eq!(
20421            waiting.message_stream_waits,
20422            vec![json!({"stream_name": "orders", "after_position": 0})]
20423        );
20424
20425        let first_delivery = delivery("message-1", 1, "one");
20426        let first_arrival = worker()
20427            .execute_workflow_task_decision(task_with_resume(
20428                vec![opened(1)],
20429                first_delivery.clone(),
20430            ))
20431            .expect("replacement worker consumes the first arrival");
20432        assert_eq!(
20433            first_arrival.commands,
20434            vec![json!({
20435                "type": "open_signal_wait",
20436                "signal_name": MESSAGE_STREAM_SIGNAL,
20437            })]
20438        );
20439        assert_eq!(
20440            first_arrival.message_stream_cursors,
20441            vec![json!({"stream_name": "orders", "through_position": 1})]
20442        );
20443        assert_eq!(
20444            first_arrival.message_stream_waits,
20445            vec![json!({"stream_name": "orders", "after_position": 1})]
20446        );
20447
20448        let second_delivery = delivery("message-2", 2, "two");
20449        let first_applied = applied(1, first_delivery);
20450        let completed = worker()
20451            .execute_workflow_task_decision(task_with_resume(
20452                vec![opened(1), first_applied.clone(), opened(2)],
20453                second_delivery.clone(),
20454            ))
20455            .expect("next replacement worker consumes the second arrival");
20456        assert_eq!(completed.commands.len(), 1);
20457        assert_eq!(completed.commands[0]["type"], "complete_workflow");
20458        assert_eq!(
20459            decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20460                .expect("workflow result"),
20461            json!(["message-1", "message-2"])
20462        );
20463        assert_eq!(
20464            completed.message_stream_cursors,
20465            vec![json!({"stream_name": "orders", "through_position": 2})]
20466        );
20467        assert!(completed.message_stream_waits.is_empty());
20468
20469        let replay_history = vec![
20470            opened(1),
20471            first_applied,
20472            opened(2),
20473            applied(2, second_delivery),
20474        ];
20475        for _cold_worker_or_restart in 0..2 {
20476            let replayed = worker()
20477                .execute_workflow_task_decision(workflow_task(
20478                    "rust.message-stream",
20479                    replay_history.clone(),
20480                    DEFAULT_CODEC,
20481                ))
20482                .expect("cold worker replays each logical message exactly once");
20483            assert_eq!(replayed.commands.len(), 1);
20484            assert_eq!(
20485                decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20486                    .expect("replayed workflow result"),
20487                json!(["message-1", "message-2"])
20488            );
20489            assert_eq!(
20490                replayed.message_stream_cursors,
20491                vec![json!({"stream_name": "orders", "through_position": 2})]
20492            );
20493            assert!(replayed.message_stream_waits.is_empty());
20494        }
20495    }
20496
20497    #[test]
20498    fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20499        assert!(!worker_protocol_supports_message_streams("1.14"));
20500        assert!(worker_protocol_supports_message_streams("1.15"));
20501        assert!(worker_protocol_supports_message_streams("1.16"));
20502        assert!(worker_protocol_supports_message_streams(
20503            WORKER_PROTOCOL_VERSION
20504        ));
20505        assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20506    }
20507
20508    #[test]
20509    fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20510        let ctx = workflow_context(vec![
20511            history_event(
20512                "ConditionWaitOpened",
20513                json!({
20514                    "sequence": 1,
20515                    "condition_wait_id": "condition:1",
20516                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20517                    "condition_key": "signal:finish",
20518                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20519                }),
20520            ),
20521            history_event(
20522                "ConditionWaitSatisfied",
20523                json!({
20524                    "sequence": 1,
20525                    "condition_wait_id": "condition:1",
20526                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20527                    "condition_key": "signal:finish",
20528                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20529                }),
20530            ),
20531            history_event(
20532                "SignalReceived",
20533                json!({"signal_name": "finish", "arguments": []}),
20534            ),
20535        ]);
20536        let mut signal = Box::pin(ctx.wait_signal("finish"));
20537        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20538
20539        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20540            signal.as_mut().poll(&mut task_context)
20541        else {
20542            panic!("condition history must not resolve as a typed signal wait");
20543        };
20544        assert_eq!(failure.reason, "recorded_command_mismatch");
20545        assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20546    }
20547
20548    #[test]
20549    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20550        let signal_then_timer = vec![
20551            history_event(
20552                "SignalWaitOpened",
20553                json!({"sequence": 1, "signal_name": "go"}),
20554            ),
20555            history_event(
20556                "SignalApplied",
20557                json!({
20558                    "sequence": 1,
20559                    "signal_name": "go",
20560                    "value": fixture_envelope(json!(["now"])),
20561                }),
20562            ),
20563            history_event(
20564                "TimerScheduled",
20565                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20566            ),
20567            history_event(
20568                "TimerFired",
20569                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20570            ),
20571        ];
20572
20573        let ctx = workflow_context(signal_then_timer.clone());
20574        let mut signal = Box::pin(ctx.wait_signal("go"));
20575        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20576        assert!(matches!(
20577            signal.as_mut().poll(&mut task_context),
20578            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20579        ));
20580        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20581        assert!(matches!(
20582            timer.as_mut().poll(&mut task_context),
20583            Poll::Ready(Ok(()))
20584        ));
20585        ctx.ensure_history_consumed()
20586            .expect("signal and timer history consumed in order");
20587
20588        let reordered = workflow_context(signal_then_timer);
20589        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20590        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20591            timer_first.as_mut().poll(&mut task_context)
20592        else {
20593            panic!("timer cannot consume signal-wait-first history");
20594        };
20595        assert_eq!(failure.reason, "recorded_command_mismatch");
20596        assert_eq!(failure.sequence, Some(1));
20597        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20598
20599        let timer_then_signal = vec![
20600            history_event(
20601                "TimerScheduled",
20602                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20603            ),
20604            history_event(
20605                "TimerFired",
20606                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20607            ),
20608            history_event(
20609                "SignalWaitOpened",
20610                json!({"sequence": 2, "signal_name": "go"}),
20611            ),
20612            history_event(
20613                "SignalApplied",
20614                json!({
20615                    "sequence": 2,
20616                    "signal_name": "go",
20617                    "value": fixture_envelope(json!([])),
20618                }),
20619            ),
20620        ];
20621        let reordered = workflow_context(timer_then_signal);
20622        let mut signal_first = Box::pin(reordered.wait_signal("go"));
20623        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20624            signal_first.as_mut().poll(&mut task_context)
20625        else {
20626            panic!("signal wait cannot consume timer-first history");
20627        };
20628        assert_eq!(failure.reason, "recorded_command_mismatch");
20629        assert_eq!(failure.sequence, Some(1));
20630        assert_eq!(failure.expected.as_deref(), Some("timer"));
20631    }
20632
20633    #[test]
20634    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20635        let duplicate_timer = WorkflowState::new(
20636            vec![
20637                history_event(
20638                    "TimerScheduled",
20639                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20640                ),
20641                history_event(
20642                    "TimerScheduled",
20643                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20644                ),
20645            ],
20646            "rust-workers".to_string(),
20647            DEFAULT_CODEC.to_string(),
20648            None,
20649        )
20650        .expect_err("one workflow sequence cannot schedule two timers");
20651        assert!(matches!(
20652            duplicate_timer,
20653            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20654                if reason == "timer_schedule_missing_or_duplicate"
20655        ));
20656
20657        let colliding_kinds = WorkflowState::new(
20658            vec![
20659                history_event(
20660                    "TimerScheduled",
20661                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20662                ),
20663                history_event(
20664                    "ActivityCompleted",
20665                    json!({"sequence": 1, "activity_type": "same-sequence"}),
20666                ),
20667            ],
20668            "rust-workers".to_string(),
20669            DEFAULT_CODEC.to_string(),
20670            None,
20671        )
20672        .expect_err("one workflow sequence cannot identify two command kinds");
20673        assert!(matches!(
20674            colliding_kinds,
20675            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20676                if reason == "durable_command_sequence_collision"
20677        ));
20678
20679        let duplicate_signal_wait = WorkflowState::new(
20680            vec![
20681                history_event(
20682                    "SignalWaitOpened",
20683                    json!({"sequence": 1, "signal_name": "go"}),
20684                ),
20685                history_event(
20686                    "SignalWaitOpened",
20687                    json!({"sequence": 1, "signal_name": "go"}),
20688                ),
20689            ],
20690            "rust-workers".to_string(),
20691            DEFAULT_CODEC.to_string(),
20692            None,
20693        )
20694        .expect_err("one workflow sequence cannot open two signal waits");
20695        assert!(matches!(
20696            duplicate_signal_wait,
20697            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20698                if reason == "signal_wait_open_missing_or_duplicate"
20699        ));
20700    }
20701
20702    #[test]
20703    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20704        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20705            .expect("side-effect result");
20706        let ctx = workflow_context(vec![history_event(
20707            "SideEffectRecorded",
20708            json!({"sequence": 99, "result": result}),
20709        )]);
20710
20711        let replayed: Value = ctx
20712            .side_effect(|| panic!("recorded side effect must not run"))
20713            .expect("positive global workflow sequence is valid");
20714        assert_eq!(replayed, json!({"captured": true}));
20715        ctx.ensure_history_consumed().expect("history consumed");
20716    }
20717
20718    #[test]
20719    fn workflow_history_rejects_zero_and_descending_command_sequences() {
20720        let result =
20721            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20722        let zero = WorkflowState::new(
20723            vec![history_event(
20724                "SideEffectRecorded",
20725                json!({"sequence": 0, "result": result.clone()}),
20726            )],
20727            "rust-workers".to_string(),
20728            DEFAULT_CODEC.to_string(),
20729            None,
20730        )
20731        .expect_err("durable command sequences must be positive");
20732        assert!(matches!(
20733            zero,
20734            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20735                if reason == "durable_command_sequence_invalid"
20736        ));
20737
20738        let descending = WorkflowState::new(
20739            vec![
20740                history_event(
20741                    "SideEffectRecorded",
20742                    json!({"sequence": 3, "result": result}),
20743                ),
20744                history_event(
20745                    "VersionMarkerRecorded",
20746                    json!({
20747                        "sequence": 2,
20748                        "change_id": "descending-marker",
20749                        "version": 1,
20750                        "min_supported": 1,
20751                        "max_supported": 1,
20752                    }),
20753                ),
20754            ],
20755            "rust-workers".to_string(),
20756            DEFAULT_CODEC.to_string(),
20757            None,
20758        )
20759        .expect_err("new durable commands must remain strictly ordered");
20760        let Error::NonDeterministicReplay(failure) = descending else {
20761            panic!("expected typed replay failure");
20762        };
20763        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20764        assert_eq!(failure.sequence, Some(2));
20765        assert_eq!(
20766            failure.expected.as_deref(),
20767            Some("workflow sequence greater than 3")
20768        );
20769        assert_eq!(failure.actual.as_deref(), Some("2"));
20770    }
20771
20772    #[test]
20773    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20774        fn worker() -> Worker {
20775            let client = Client::new("http://127.0.0.1:8080").expect("client");
20776            let mut worker = Worker::new(client, "rust-workers");
20777            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20778                ctx.wait_signal("finish").await?;
20779                let marker: String =
20780                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20781                assert_eq!(marker, "after-finish");
20782                Ok(json!("finished"))
20783            });
20784            worker
20785        }
20786
20787        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20788            .expect("side-effect result");
20789        let task = workflow_task(
20790            "rust.finish-after-gaps",
20791            vec![
20792                history_event(
20793                    "SignalWaitOpened",
20794                    json!({"sequence": 1, "signal_name": "finish"}),
20795                ),
20796                history_event(
20797                    "SignalReceived",
20798                    json!({
20799                        "signal_id": "increment-3",
20800                        "signal_name": "increment",
20801                        "workflow_sequence": 2,
20802                        "payload_codec": DEFAULT_CODEC,
20803                        "arguments": fixture_envelope(json!([3])),
20804                    }),
20805                ),
20806                history_event(
20807                    "SignalReceived",
20808                    json!({
20809                        "signal_id": "increment-5",
20810                        "signal_name": "increment",
20811                        "workflow_sequence": 3,
20812                        "payload_codec": DEFAULT_CODEC,
20813                        "arguments": fixture_envelope(json!([5])),
20814                    }),
20815                ),
20816                history_event(
20817                    "SignalReceived",
20818                    json!({
20819                        "signal_id": "finish",
20820                        "signal_name": "finish",
20821                        "workflow_sequence": 4,
20822                        "payload_codec": DEFAULT_CODEC,
20823                        "arguments": fixture_envelope(json!([])),
20824                    }),
20825                ),
20826                history_event(
20827                    "SignalApplied",
20828                    json!({
20829                        "sequence": 1,
20830                        "signal_id": "finish",
20831                        "signal_name": "finish",
20832                        "payload_codec": DEFAULT_CODEC,
20833                        "value": fixture_envelope(json!([])),
20834                    }),
20835                ),
20836                history_event(
20837                    "SideEffectRecorded",
20838                    json!({"sequence": 5, "result": marker}),
20839                ),
20840            ],
20841            DEFAULT_CODEC,
20842        );
20843
20844        for _original_or_cold_worker in 0..2 {
20845            let commands = worker()
20846                .execute_workflow_task(task.clone())
20847                .expect("signal gaps preserve deterministic replay");
20848            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20849            assert_eq!(commands[0]["type"], "complete_workflow");
20850            assert_eq!(
20851                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20852                json!("finished")
20853            );
20854        }
20855    }
20856
20857    #[test]
20858    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20859        let ctx = workflow_context(Vec::new());
20860        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20861        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20862        assert!(matches!(
20863            sleep.as_mut().poll(&mut task_context),
20864            Poll::Ready(Err(Error::TimerDurationOverflow))
20865        ));
20866        assert!(ctx.take_commands().expect("commands").is_empty());
20867    }
20868
20869    #[test]
20870    fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20871        let entries = AvroValue::Map(BTreeMap::from([
20872            ("text".to_string(), AvroValue::String("same".to_string())),
20873            (
20874                "nested".to_string(),
20875                AvroValue::Map(BTreeMap::from([
20876                    ("beta".to_string(), AvroValue::Long(2)),
20877                    ("alpha".to_string(), AvroValue::Long(1)),
20878                ])),
20879            ),
20880            ("long".to_string(), AvroValue::Long(7)),
20881            ("double".to_string(), AvroValue::Double(7.0)),
20882            ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20883        ]));
20884        let ctx = workflow_context(Vec::new());
20885        ctx.upsert_memo(entries.clone()).expect("valid memo update");
20886        let commands = ctx.take_commands().expect("commands");
20887
20888        assert_eq!(commands.len(), 1);
20889        assert_eq!(commands[0]["type"], "upsert_memo");
20890        let server_entries = json!({
20891            "codec": "avro",
20892            "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20893        });
20894        assert_eq!(
20895            commands[0]["entries"]
20896                .as_object()
20897                .expect("entries envelope")
20898                .keys()
20899                .collect::<Vec<_>>(),
20900            vec!["blob", "codec"]
20901        );
20902        assert_eq!(commands[0]["entries"], server_entries);
20903        let wire_entries =
20904            decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20905        assert_eq!(wire_entries, entries);
20906
20907        let history = vec![history_event(
20908            "MemoUpserted",
20909            json!({
20910                "sequence": 1,
20911                "entries": server_entries.clone(),
20912                "merged": server_entries,
20913            }),
20914        )];
20915        let replay = workflow_context(history.clone());
20916        replay
20917            .upsert_memo(entries.clone())
20918            .expect("matching replay identity");
20919        assert!(replay.take_commands().expect("replay commands").is_empty());
20920
20921        let changed_types = AvroValue::Map(BTreeMap::from([
20922            ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20923            (
20924                "nested".to_string(),
20925                AvroValue::Map(BTreeMap::from([
20926                    ("alpha".to_string(), AvroValue::Long(1)),
20927                    ("beta".to_string(), AvroValue::Long(2)),
20928                ])),
20929            ),
20930            ("long".to_string(), AvroValue::Double(7.0)),
20931            ("double".to_string(), AvroValue::Long(7)),
20932            ("binary".to_string(), AvroValue::String("same".to_string())),
20933        ]));
20934        let error = workflow_context(history)
20935            .upsert_memo(changed_types)
20936            .expect_err("memo replay identity must preserve Avro value types");
20937        assert!(matches!(
20938            error,
20939            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20940        ));
20941    }
20942
20943    #[test]
20944    fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20945        let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20946            .expect("memo envelope");
20947        let replay = workflow_context(vec![history_event(
20948            "MemoUpserted",
20949            json!({
20950                "sequence": 1,
20951                "entries": original.clone(),
20952                "merged": original
20953            }),
20954        )]);
20955        let error = replay
20956            .upsert_memo(json!({"stage": "changed"}))
20957            .expect_err("changed memo update must fail replay");
20958        assert!(matches!(
20959            error,
20960            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20961        ));
20962
20963        let invalid = workflow_context(Vec::new())
20964            .upsert_memo(
20965                json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20966            )
20967            .expect_err("oversized key");
20968        assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20969    }
20970
20971    #[test]
20972    fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20973        let negative_zero = AvroValue::Map(BTreeMap::from([(
20974            "reading".to_string(),
20975            AvroValue::Double(-0.0),
20976        )]));
20977        let negative_zero_envelope =
20978            encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20979        let history = vec![history_event(
20980            "MemoUpserted",
20981            json!({
20982                "sequence": 1,
20983                "entries": negative_zero_envelope.clone(),
20984                "merged": negative_zero_envelope,
20985            }),
20986        )];
20987
20988        workflow_context(history.clone())
20989            .upsert_memo(negative_zero)
20990            .expect("matching negative-zero history identity");
20991
20992        let error = workflow_context(history)
20993            .upsert_memo(AvroValue::Map(BTreeMap::from([(
20994                "reading".to_string(),
20995                AvroValue::Double(0.0),
20996            )])))
20997            .expect_err("positive zero must not consume negative-zero memo history");
20998        assert!(matches!(
20999            error,
21000            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
21001        ));
21002    }
21003
21004    #[test]
21005    fn workflow_memo_capability_requires_flag_and_command_advertisement() {
21006        let supported = json!({
21007            "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
21008            "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
21009        });
21010        assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
21011        assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
21012            "workflow_memo_updates": {"supported": false},
21013            "supported_workflow_task_commands": ["upsert_memo"]
21014        }))));
21015        assert!(commands_use_workflow_memo_updates(&[json!({
21016            "type": "upsert_memo",
21017            "entries": {"stage": "processing"}
21018        })]));
21019    }
21020
21021    #[test]
21022    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
21023        let client = Client::new("http://127.0.0.1:8080").expect("client");
21024        let mut worker = Worker::new(client, "rust-workers");
21025        worker.register_workflow("rust.timer", |ctx, _input| async move {
21026            ctx.sleep(Duration::from_secs(5)).await?;
21027            ctx.activity("after-timer", json!([])).await
21028        });
21029
21030        let task = |history_events| WorkflowTask {
21031            task_id: "wft-rust-timer-1".to_string(),
21032            workflow_command_id: None,
21033            workflow_id: Some("wf-rust-timer".to_string()),
21034            run_id: Some("run-rust-timer".to_string()),
21035            workflow_type: "rust.timer".to_string(),
21036            cancel_requested: false,
21037            payload_codec: DEFAULT_CODEC.to_string(),
21038            arguments: Some(
21039                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21040            ),
21041            history_events,
21042            total_history_events: None,
21043            history_size_bytes: None,
21044            continue_as_new_recommended: None,
21045            history_budget_pressure: None,
21046            next_history_page_token: None,
21047            workflow_task_attempt: 1,
21048            workflow_signal_id: None,
21049            signal_name: None,
21050            signal_arguments: None,
21051            workflow_update_id: None,
21052            update_name: None,
21053            lease_owner: Some("rust-worker".to_string()),
21054        };
21055
21056        let initial = worker
21057            .execute_workflow_task(task(Vec::new()))
21058            .expect("initial timer task");
21059        assert_eq!(
21060            initial,
21061            vec![json!({"type": "start_timer", "delay_seconds": 5})]
21062        );
21063
21064        let activity_result =
21065            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
21066        let replayed = worker
21067            .execute_workflow_task(task(vec![
21068                history_event(
21069                    "TimerScheduled",
21070                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21071                ),
21072                history_event(
21073                    "TimerFired",
21074                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21075                ),
21076                history_event(
21077                    "ActivityCompleted",
21078                    json!({
21079                        "sequence": 2,
21080                        "activity_type": "after-timer",
21081                        "payload_codec": DEFAULT_CODEC,
21082                        "result": activity_result,
21083                    }),
21084                ),
21085            ]))
21086            .expect("replayed workflow task");
21087        assert_eq!(replayed.len(), 1);
21088        assert_eq!(replayed[0]["type"], "complete_workflow");
21089        assert_eq!(
21090            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
21091            json!("done")
21092        );
21093    }
21094
21095    #[test]
21096    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
21097        let client = Client::new("http://127.0.0.1:8080").expect("client");
21098        let mut worker = Worker::new(client, "rust-workers");
21099        worker.register_workflow("rust.continue", |ctx, _input| async move {
21100            ctx.continue_as_new_with_options(
21101                ContinueAsNewOptions::new()
21102                    .workflow_type("rust.next")
21103                    .task_queue("next-workers"),
21104                json!([2, {"cursor": "next"}]),
21105            )
21106        });
21107
21108        let commands = worker
21109            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
21110            .expect("continue-as-new command");
21111
21112        assert_eq!(commands.len(), 1);
21113        assert_eq!(commands[0]["type"], "continue_as_new");
21114        assert_eq!(commands[0]["workflow_type"], "rust.next");
21115        assert_eq!(commands[0]["queue"], "next-workers");
21116        assert_eq!(
21117            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
21118                .expect("continue-as-new arguments"),
21119            json!([2, {"cursor": "next"}])
21120        );
21121    }
21122
21123    #[test]
21124    fn continue_as_new_preserves_typed_arguments() {
21125        let client = Client::new("http://127.0.0.1:8080").expect("client");
21126        let mut worker = Worker::new(client, "rust-workers");
21127        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
21128            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
21129            unreachable!("continue-as-new returns a control-flow error")
21130        });
21131
21132        let commands = worker
21133            .execute_workflow_task(workflow_task(
21134                "rust.typed-continue",
21135                Vec::new(),
21136                DEFAULT_CODEC,
21137            ))
21138            .expect("typed continue-as-new command");
21139
21140        assert_eq!(commands[0]["type"], "continue_as_new");
21141        assert_eq!(
21142            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
21143                .expect("typed continue arguments"),
21144            AvroValue::Array(vec![typed_fidelity_probe()])
21145        );
21146    }
21147
21148    #[test]
21149    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
21150        let client = Client::new("http://127.0.0.1:8080").expect("client");
21151        let mut worker = Worker::new(client, "rust-workers");
21152        worker.register_workflow("rust.continue", |ctx, _input| async move {
21153            ctx.continue_as_new(json!([2]))
21154        });
21155        let task = workflow_task(
21156            "rust.continue",
21157            vec![history_event(
21158                "WorkflowContinuedAsNew",
21159                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
21160            )],
21161            DEFAULT_CODEC,
21162        );
21163
21164        for _worker_restart_or_redelivery in 0..2 {
21165            let commands = worker
21166                .execute_workflow_task(task.clone())
21167                .expect("recorded transition replays");
21168            assert!(
21169                commands.is_empty(),
21170                "replay must not emit another successor"
21171            );
21172        }
21173    }
21174
21175    #[test]
21176    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
21177        let ctx = workflow_context(Vec::new());
21178        let error = ctx
21179            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
21180            .expect_err("blank queue must be rejected");
21181
21182        let Error::InvalidContinueAsNewOptions(error) = error else {
21183            panic!("expected typed continue-as-new validation error");
21184        };
21185        assert_eq!(error.field, "task_queue");
21186        assert!(ctx.take_commands().expect("commands").is_empty());
21187    }
21188
21189    #[test]
21190    fn workflow_context_exposes_server_history_budget() {
21191        let client = Client::new("http://127.0.0.1:8080").expect("client");
21192        let mut worker = Worker::new(client, "rust-workers");
21193        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
21194            let budget = ctx.history_budget()?;
21195            Ok(json!({
21196                "events": budget.event_count,
21197                "bytes": budget.size_bytes,
21198                "recommended": budget.continue_as_new_recommended,
21199                "pressure": budget.pressure,
21200            }))
21201        });
21202        let task: WorkflowTask = serde_json::from_value(json!({
21203            "task_id": "task-history-budget",
21204            "workflow_type": "rust.history-budget",
21205            "payload_codec": DEFAULT_CODEC,
21206            "history_events": [],
21207            "total_history_events": 480,
21208            "history_size_bytes": 1_048_576,
21209            "continue_as_new_recommended": true,
21210            "history_budget_pressure": "continue_as_new_recommended",
21211        }))
21212        .expect("published workflow task");
21213
21214        let commands = worker
21215            .execute_workflow_task(task)
21216            .expect("history-budget workflow");
21217        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
21218        assert_eq!(result["events"], 480);
21219        assert_eq!(result["bytes"], 1_048_576);
21220        assert_eq!(result["recommended"], true);
21221        assert_eq!(result["pressure"], "continue_as_new_recommended");
21222    }
21223
21224    #[test]
21225    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
21226        let client = Client::new("http://127.0.0.1:8080").expect("client");
21227        let mut worker = Worker::new(client, "rust-workers");
21228        worker.register_workflow("rust.failing", |_ctx, _input| async move {
21229            Err(Error::Codec("rust_conformance_failure".to_string()))
21230        });
21231        let task = WorkflowTask {
21232            task_id: "wft-rust-failing-1".to_string(),
21233            workflow_command_id: None,
21234            workflow_id: Some("wf-rust-failing".to_string()),
21235            run_id: Some("run-rust-failing".to_string()),
21236            workflow_type: "rust.failing".to_string(),
21237            cancel_requested: false,
21238            payload_codec: DEFAULT_CODEC.to_string(),
21239            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21240            history_events: Vec::new(),
21241            total_history_events: Some(0),
21242            history_size_bytes: None,
21243            continue_as_new_recommended: None,
21244            history_budget_pressure: None,
21245            next_history_page_token: None,
21246            workflow_task_attempt: 1,
21247            workflow_signal_id: None,
21248            signal_name: None,
21249            signal_arguments: None,
21250            workflow_update_id: None,
21251            update_name: None,
21252            lease_owner: Some("rust-worker".to_string()),
21253        };
21254
21255        let commands = worker
21256            .execute_workflow_task(task)
21257            .expect("handler failure becomes a workflow command");
21258
21259        assert_eq!(commands.len(), 1);
21260        assert_eq!(commands[0]["type"], "fail_workflow");
21261        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
21262        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
21263        assert_eq!(commands[0]["non_retryable"], false);
21264        assert_eq!(
21265            commands[0]["message"],
21266            "codec error: rust_conformance_failure"
21267        );
21268        assert_eq!(
21269            commands[0]["exception"]["message"],
21270            "codec error: rust_conformance_failure"
21271        );
21272    }
21273
21274    #[test]
21275    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
21276        let client = Client::new("http://127.0.0.1:8080").expect("client");
21277        let mut worker = Worker::new(client, "rust-workers");
21278        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
21279            let _: String = ctx.side_effect(|| "captured".to_string())?;
21280            Err(Error::WorkerLoop("application failure".to_string()))
21281        });
21282
21283        let commands = worker
21284            .execute_workflow_task(workflow_task(
21285                "rust.failing-after-side-effect",
21286                Vec::new(),
21287                DEFAULT_CODEC,
21288            ))
21289            .expect("ordinary failure remains a workflow decision");
21290
21291        assert_eq!(commands.len(), 2);
21292        assert_eq!(commands[0]["type"], "record_side_effect");
21293        assert_eq!(commands[1]["type"], "fail_workflow");
21294    }
21295
21296    #[test]
21297    fn uncaught_recorded_activity_failure_claims_only_its_persisted_boundary() {
21298        let client = Client::new("http://127.0.0.1:8080").expect("client");
21299        let mut worker = Worker::new(client, "rust-workers");
21300        worker.register_workflow("rust.redrive", |ctx, _input| async move {
21301            ctx.activity("greet", json!([])).await?;
21302            Ok(Value::Null)
21303        });
21304        worker.register_workflow("rust.redrive-translated", |ctx, _input| async move {
21305            match ctx.activity("greet", json!([])).await {
21306                Err(Error::ActivityFailed(_)) => Err(Error::WorkerLoop("translated".to_string())),
21307                result => result,
21308            }
21309        });
21310
21311        let failed = history_event(
21312            "ActivityFailed",
21313            json!({
21314                "sequence": 1,
21315                "activity_type": "greet",
21316                "activity_execution_id": "activity-1",
21317                "message": "failed"
21318            }),
21319        );
21320        let commands = worker
21321            .execute_workflow_task(workflow_task(
21322                "rust.redrive",
21323                vec![failed.clone()],
21324                DEFAULT_CODEC,
21325            ))
21326            .expect("recorded failure becomes a workflow command");
21327        assert_eq!(commands[0]["type"], "fail_workflow");
21328        assert_eq!(commands[0]["failed_step_sequence"], 1);
21329        assert_eq!(commands[0]["failed_activity_execution_id"], "activity-1");
21330
21331        let translated = worker
21332            .execute_workflow_task(workflow_task(
21333                "rust.redrive-translated",
21334                vec![failed],
21335                DEFAULT_CODEC,
21336            ))
21337            .expect("translated failure becomes a workflow command");
21338        assert_eq!(translated[0]["type"], "fail_workflow");
21339        assert!(translated[0].get("failed_step_sequence").is_none());
21340        assert!(translated[0].get("failed_activity_execution_id").is_none());
21341
21342        let without_identity = worker
21343            .execute_workflow_task(workflow_task(
21344                "rust.redrive",
21345                vec![history_event(
21346                    "ActivityFailed",
21347                    json!({"sequence": 1, "activity_type": "greet", "message": "failed"}),
21348                )],
21349                DEFAULT_CODEC,
21350            ))
21351            .expect("incomplete identity still fails the workflow");
21352        assert!(without_identity[0].get("failed_step_sequence").is_none());
21353
21354        let timed_out = worker
21355            .execute_workflow_task(workflow_task(
21356                "rust.redrive",
21357                vec![history_event(
21358                    "ActivityTimedOut",
21359                    json!({
21360                        "sequence": 1,
21361                        "activity_type": "greet",
21362                        "activity_execution_id": "activity-timeout",
21363                    }),
21364                )],
21365                DEFAULT_CODEC,
21366            ))
21367            .expect("timeout still fails the workflow");
21368        assert!(timed_out[0].get("failed_step_sequence").is_none());
21369    }
21370
21371    #[test]
21372    fn redriven_history_reuses_completed_prefix_and_reschedules_only_failed_step() {
21373        let client = Client::new("http://127.0.0.1:8080").expect("client");
21374        let mut worker = Worker::new(client, "rust-workers");
21375        worker.register_workflow("rust.redrive-successor", |ctx, _input| async move {
21376            let first = ctx.activity("first", json!([])).await?;
21377            let second = ctx.activity("second", json!([first.clone()])).await?;
21378            Ok(json!({"first": first, "second": second}))
21379        });
21380        let first = history_event(
21381            "ActivityCompleted",
21382            json!({
21383                "sequence": 1,
21384                "activity_type": "first",
21385                "result": encode_value_envelope(&json!("recorded"), DEFAULT_CODEC).expect("first result"),
21386                "payload_codec": DEFAULT_CODEC,
21387                "reused_from_run_id": "failed-run",
21388                "reused_activity_execution_id": "original-first",
21389            }),
21390        );
21391
21392        let retry = worker
21393            .execute_workflow_task(workflow_task(
21394                "rust.redrive-successor",
21395                vec![first.clone()],
21396                DEFAULT_CODEC,
21397            ))
21398            .expect("reused result replays before failed step");
21399        assert_eq!(retry.len(), 1);
21400        assert_eq!(retry[0]["type"], "schedule_activity");
21401        assert_eq!(retry[0]["activity_type"], "second");
21402        assert_eq!(
21403            decode_wire_value(&retry[0]["arguments"], DEFAULT_CODEC).expect("second arguments"),
21404            json!(["recorded"]),
21405        );
21406
21407        let second = history_event(
21408            "ActivityCompleted",
21409            json!({
21410                "sequence": 2,
21411                "activity_type": "second",
21412                "result": encode_value_envelope(&json!("retried"), DEFAULT_CODEC).expect("second result"),
21413                "payload_codec": DEFAULT_CODEC,
21414            }),
21415        );
21416        let completed = worker
21417            .execute_workflow_task(workflow_task(
21418                "rust.redrive-successor",
21419                vec![first, second],
21420                DEFAULT_CODEC,
21421            ))
21422            .expect("retried step completes the successor");
21423        assert_eq!(completed.len(), 1);
21424        assert_eq!(completed[0]["type"], "complete_workflow");
21425        assert_eq!(
21426            decode_wire_value(&completed[0]["result"], DEFAULT_CODEC).expect("workflow result"),
21427            json!({"first": "recorded", "second": "retried"}),
21428        );
21429    }
21430
21431    #[test]
21432    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
21433        let client = Client::new("http://127.0.0.1:8080").expect("client");
21434        let mut worker = Worker::new(client, "rust-workers");
21435        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
21436            Err(Error::WorkerLoop("application failure".to_string()))
21437        });
21438        let result =
21439            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
21440
21441        let error = worker
21442            .execute_workflow_task(workflow_task(
21443                "rust.removed-side-effect",
21444                vec![history_event(
21445                    "SideEffectRecorded",
21446                    json!({"sequence": 1, "result": result}),
21447                )],
21448                DEFAULT_CODEC,
21449            ))
21450            .expect_err("removed committed history must not become fail_workflow");
21451
21452        let Error::NonDeterministicReplay(failure) = error else {
21453            panic!("expected typed replay failure");
21454        };
21455        assert_eq!(failure.reason, "recorded_commands_unconsumed");
21456        assert_eq!(failure.sequence, Some(1));
21457        assert_eq!(failure.expected.as_deref(), Some("side effect"));
21458    }
21459
21460    #[test]
21461    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
21462        let client = Client::new("http://127.0.0.1:8080").expect("client");
21463        let mut worker = Worker::new(client, "rust-workers");
21464        worker.register_workflow(
21465            "rust.side-effect-before-marker-error",
21466            |ctx, _input| async move {
21467                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
21468                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
21469                ctx.get_version("restart-safe", 2, 2)?;
21470                Ok(Value::Null)
21471            },
21472        );
21473
21474        let error = worker
21475            .execute_workflow_task(workflow_task(
21476                "rust.side-effect-before-marker-error",
21477                vec![history_event(
21478                    "VersionMarkerRecorded",
21479                    json!({
21480                        "sequence": 1,
21481                        "change_id": "restart-safe",
21482                        "version": 1,
21483                        "min_supported": 1,
21484                        "max_supported": 1,
21485                    }),
21486                )],
21487                DEFAULT_CODEC,
21488            ))
21489            .expect_err("replay error must return no queued workflow commands");
21490
21491        let Error::NonDeterministicReplay(failure) = error else {
21492            panic!("expected typed replay failure");
21493        };
21494        assert_eq!(failure.reason, "version_marker_incompatible_range");
21495        assert_eq!(failure.sequence, Some(1));
21496    }
21497
21498    #[test]
21499    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
21500        let client = Client::new("http://127.0.0.1:8080").expect("client");
21501        let mut worker = Worker::new(client, "rust-workers");
21502        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
21503            ctx.sleep(Duration::from_secs(5)).await?;
21504            Ok(json!({"status": "timer fired"}))
21505        });
21506
21507        let task = WorkflowTask {
21508            task_id: "wft-rust-timer-pending".to_string(),
21509            workflow_command_id: None,
21510            workflow_id: Some("wf-rust-timer".to_string()),
21511            run_id: Some("run-rust-timer".to_string()),
21512            workflow_type: "rust.timer.pending".to_string(),
21513            cancel_requested: false,
21514            payload_codec: DEFAULT_CODEC.to_string(),
21515            arguments: Some(
21516                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21517            ),
21518            history_events: vec![history_event(
21519                "TimerScheduled",
21520                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21521            )],
21522            total_history_events: Some(1),
21523            history_size_bytes: None,
21524            continue_as_new_recommended: None,
21525            history_budget_pressure: None,
21526            next_history_page_token: None,
21527            workflow_task_attempt: 1,
21528            workflow_signal_id: None,
21529            signal_name: None,
21530            signal_arguments: None,
21531            workflow_update_id: None,
21532            update_name: None,
21533            lease_owner: Some("rust-worker".to_string()),
21534        };
21535
21536        for _redelivery_or_restart in 0..2 {
21537            let commands = worker
21538                .execute_workflow_task(task.clone())
21539                .expect("recorded timer remains pending");
21540            assert!(
21541                commands.is_empty(),
21542                "recorded timer must not be rescheduled"
21543            );
21544        }
21545    }
21546
21547    #[test]
21548    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21549        let client = Client::new("http://127.0.0.1:8080").expect("client");
21550        let mut worker = Worker::new(client, "rust-workers");
21551        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21552            Ok(json!({"status": "completed"}))
21553        });
21554        let task = WorkflowTask {
21555            task_id: "wft-rust-timer-removed".to_string(),
21556            workflow_command_id: None,
21557            workflow_id: Some("wf-rust-timer".to_string()),
21558            run_id: Some("run-rust-timer".to_string()),
21559            workflow_type: "rust.timer.removed".to_string(),
21560            cancel_requested: false,
21561            payload_codec: DEFAULT_CODEC.to_string(),
21562            arguments: Some(
21563                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21564            ),
21565            history_events: vec![
21566                history_event(
21567                    "TimerScheduled",
21568                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21569                ),
21570                history_event(
21571                    "TimerFired",
21572                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21573                ),
21574            ],
21575            total_history_events: Some(2),
21576            history_size_bytes: None,
21577            continue_as_new_recommended: None,
21578            history_budget_pressure: None,
21579            next_history_page_token: None,
21580            workflow_task_attempt: 1,
21581            workflow_signal_id: None,
21582            signal_name: None,
21583            signal_arguments: None,
21584            workflow_update_id: None,
21585            update_name: None,
21586            lease_owner: Some("rust-worker".to_string()),
21587        };
21588
21589        let Error::NonDeterministicReplay(failure) = worker
21590            .execute_workflow_task(task)
21591            .expect_err("removed timer must fail replay")
21592        else {
21593            panic!("expected typed replay failure");
21594        };
21595        assert_eq!(failure.reason, "recorded_commands_unconsumed");
21596        assert_eq!(failure.sequence, Some(1));
21597    }
21598
21599    #[test]
21600    fn workflow_context_emits_explicit_child_workflow_contract() {
21601        let ctx = WorkflowContext {
21602            state: Arc::new(Mutex::new(
21603                WorkflowState::new_with_identity(
21604                    Vec::new(),
21605                    Some("wf-parent".to_string()),
21606                    Some("run-parent".to_string()),
21607                    "parent-workers".to_string(),
21608                    DEFAULT_CODEC.to_string(),
21609                    None,
21610                )
21611                .expect("workflow state"),
21612            )),
21613        };
21614        let options = ChildWorkflowOptions::new("python-workers")
21615            .parent_close_policy(ParentClosePolicy::RequestCancel)
21616            .retry_policy(ChildWorkflowRetryPolicy {
21617                max_attempts: Some(3),
21618                backoff_seconds: vec![1, 5],
21619                non_retryable_error_types: vec!["ValidationError".to_string()],
21620            })
21621            .execution_timeout_seconds(600)
21622            .run_timeout_seconds(120);
21623        let mut call = Box::pin(ctx.start_child_workflow(
21624            "python.fulfil-order",
21625            options,
21626            json!([{"order_id": "order-42"}]),
21627        ));
21628        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21629
21630        assert!(matches!(
21631            call.as_mut().poll(&mut task_context),
21632            Poll::Pending
21633        ));
21634        let commands = ctx.take_commands().expect("commands");
21635        assert_eq!(commands.len(), 1);
21636        let command = &commands[0];
21637        assert_eq!(command["type"], "start_child_workflow");
21638        assert_eq!(command["workflow_type"], "python.fulfil-order");
21639        assert_eq!(command["queue"], "python-workers");
21640        assert_eq!(command["parent_close_policy"], "request_cancel");
21641        assert_eq!(command["retry_policy"]["max_attempts"], 3);
21642        assert_eq!(command["execution_timeout_seconds"], 600);
21643        assert_eq!(command["run_timeout_seconds"], 120);
21644        assert_eq!(
21645            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21646            json!([{"order_id": "order-42"}])
21647        );
21648    }
21649
21650    fn child_parent_worker() -> Worker {
21651        let client = Client::new("http://127.0.0.1:8080").expect("client");
21652        let mut worker = Worker::new(client, "rust-parent-workers");
21653        worker.register_workflow("rust.parent", |ctx, _input| async move {
21654            let child = ctx
21655                .start_child_workflow(
21656                    "python.child",
21657                    ChildWorkflowOptions::new("python-child-workers")
21658                        .parent_close_policy(ParentClosePolicy::Terminate),
21659                    json!([{"codec_probe": [1, true, "rust"]}]),
21660                )
21661                .await?;
21662            Ok(json!({
21663                "parent_workflow_id": child.parent.workflow_id,
21664                "parent_run_id": child.parent.run_id,
21665                "child_workflow_id": child.child.workflow_id,
21666                "child_run_id": child.child.run_id,
21667                "child_workflow_type": child.child_workflow_type,
21668                "result": child.result,
21669            }))
21670        });
21671        worker
21672    }
21673
21674    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21675        WorkflowTask {
21676            task_id: "wft-child-parent".to_string(),
21677            workflow_command_id: None,
21678            workflow_id: Some("wf-parent".to_string()),
21679            run_id: Some("run-parent".to_string()),
21680            workflow_type: "rust.parent".to_string(),
21681            cancel_requested: false,
21682            payload_codec: DEFAULT_CODEC.to_string(),
21683            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21684            history_events: vec![
21685                HistoryEvent {
21686                    event_type: "ChildWorkflowScheduled".to_string(),
21687                    payload: json!({
21688                        "sequence": 1,
21689                        "child_call_id": "call-child",
21690                        "child_workflow_instance_id": "wf-child",
21691                        "child_workflow_run_id": "run-child",
21692                        "child_workflow_type": "python.child",
21693                    }),
21694                    raw: HashMap::new(),
21695                },
21696                HistoryEvent {
21697                    event_type: event_type.to_string(),
21698                    payload,
21699                    raw: HashMap::new(),
21700                },
21701            ],
21702            total_history_events: Some(2),
21703            history_size_bytes: None,
21704            continue_as_new_recommended: None,
21705            history_budget_pressure: None,
21706            next_history_page_token: None,
21707            workflow_task_attempt: 1,
21708            workflow_signal_id: None,
21709            signal_name: None,
21710            signal_arguments: None,
21711            workflow_update_id: None,
21712            update_name: None,
21713            lease_owner: Some("rust-worker".to_string()),
21714        }
21715    }
21716
21717    #[test]
21718    fn committed_child_result_replays_without_starting_a_duplicate() {
21719        let worker = child_parent_worker();
21720        let task = child_parent_task(
21721            "ChildRunCompleted",
21722            json!({
21723                "sequence": 1,
21724                "child_call_id": "call-child",
21725                "child_workflow_instance_id": "wf-child",
21726                "child_workflow_run_id": "run-child",
21727                "child_workflow_type": "python.child",
21728                "payload_codec": DEFAULT_CODEC,
21729                "result": fixture_envelope(json!({"from":"python","ok":true})),
21730            }),
21731        );
21732
21733        for _restart in 0..2 {
21734            let commands = worker
21735                .execute_workflow_task(task.clone())
21736                .expect("replayed parent task");
21737            assert_eq!(commands.len(), 1);
21738            assert_eq!(commands[0]["type"], "complete_workflow");
21739            assert!(!commands
21740                .iter()
21741                .any(|command| command["type"] == "start_child_workflow"));
21742            let output =
21743                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21744            assert_eq!(output["parent_workflow_id"], "wf-parent");
21745            assert_eq!(output["parent_run_id"], "run-parent");
21746            assert_eq!(output["child_workflow_id"], "wf-child");
21747            assert_eq!(output["child_run_id"], "run-child");
21748            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21749        }
21750    }
21751
21752    #[test]
21753    fn typed_child_arguments_and_results_survive_replay() {
21754        let client = Client::new("http://127.0.0.1:8080").expect("client");
21755        let mut worker = Worker::new(client, "rust-parent-workers");
21756        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21757            let child = ctx
21758                .start_child_workflow_avro_value(
21759                    "python.typed-child",
21760                    ChildWorkflowOptions::new("python-workers"),
21761                    AvroValue::Array(vec![typed_fidelity_probe()]),
21762                )
21763                .await?;
21764            Ok(child.result)
21765        });
21766
21767        let initial = worker
21768            .execute_workflow_task(workflow_task(
21769                "rust.typed-parent",
21770                Vec::new(),
21771                DEFAULT_CODEC,
21772            ))
21773            .expect("typed child start");
21774        assert_eq!(initial[0]["type"], "start_child_workflow");
21775        assert_eq!(
21776            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21777                .expect("typed child arguments"),
21778            AvroValue::Array(vec![typed_fidelity_probe()])
21779        );
21780
21781        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21782            .expect("typed child result");
21783        let task = workflow_task(
21784            "rust.typed-parent",
21785            vec![
21786                history_event(
21787                    "ChildWorkflowScheduled",
21788                    json!({
21789                        "sequence": 1,
21790                        "child_call_id": "call-typed",
21791                        "child_workflow_instance_id": "wf-child",
21792                        "child_workflow_run_id": "run-child",
21793                        "child_workflow_type": "python.typed-child",
21794                    }),
21795                ),
21796                history_event(
21797                    "ChildRunCompleted",
21798                    json!({
21799                        "sequence": 1,
21800                        "child_call_id": "call-typed",
21801                        "child_workflow_instance_id": "wf-child",
21802                        "child_workflow_run_id": "run-child",
21803                        "child_workflow_type": "python.typed-child",
21804                        "payload_codec": DEFAULT_CODEC,
21805                        "result": result,
21806                    }),
21807                ),
21808            ],
21809            DEFAULT_CODEC,
21810        );
21811
21812        let commands = worker
21813            .execute_workflow_task(task)
21814            .expect("typed child replay");
21815        assert_eq!(commands[0]["type"], "complete_workflow");
21816        assert_eq!(
21817            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21818                .expect("typed parent result"),
21819            typed_fidelity_probe()
21820        );
21821    }
21822
21823    #[test]
21824    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21825        let worker = child_parent_worker();
21826        let mut task = child_parent_task("unused", Value::Null);
21827        task.history_events.truncate(1);
21828        task.total_history_events = Some(1);
21829
21830        for _redelivery_or_restart in 0..2 {
21831            let commands = worker
21832                .execute_workflow_task(task.clone())
21833                .expect("recorded child remains pending");
21834            assert!(
21835                commands.is_empty(),
21836                "recorded pending child must not be started again"
21837            );
21838        }
21839    }
21840
21841    #[test]
21842    fn child_cancellation_becomes_stable_parent_failure_command() {
21843        let worker = child_parent_worker();
21844        let task = child_parent_task(
21845            "ChildRunCancelled",
21846            json!({
21847                "sequence": 1,
21848                "child_workflow_instance_id": "wf-child",
21849                "child_workflow_run_id": "run-child",
21850                "child_workflow_type": "python.child",
21851                "failure_id": "failure-child",
21852                "failure_category": "cancelled",
21853                "message": "cancelled by parent-close policy",
21854            }),
21855        );
21856
21857        let commands = worker
21858            .execute_workflow_task(task)
21859            .expect("parent settlement");
21860        assert_eq!(commands.len(), 1);
21861        assert_eq!(commands[0]["type"], "fail_workflow");
21862        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21863        assert_eq!(
21864            commands[0]["exception"]["properties"]["reason"],
21865            "cancelled"
21866        );
21867        assert_eq!(
21868            commands[0]["exception"]["properties"]["child_workflow_run_id"],
21869            "run-child"
21870        );
21871    }
21872
21873    #[test]
21874    fn workflow_can_handle_typed_child_failure() {
21875        let client = Client::new("http://127.0.0.1:8080").expect("client");
21876        let mut worker = Worker::new(client, "rust-parent-workers");
21877        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21878            match ctx
21879                .start_child_workflow(
21880                    "python.child",
21881                    ChildWorkflowOptions::new("python-child-workers"),
21882                    json!([]),
21883                )
21884                .await
21885            {
21886                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21887                    "reason": failure.reason,
21888                    "failure_id": failure.failure_id,
21889                    "exception_class": failure.exception_class,
21890                    "child_run_id": failure.child_workflow_run_id,
21891                })),
21892                Err(error) => Err(error),
21893                Ok(_) => Err(Error::WorkerLoop(
21894                    "child unexpectedly succeeded".to_string(),
21895                )),
21896            }
21897        });
21898        let mut task = child_parent_task(
21899            "ChildRunFailed",
21900            json!({
21901                "sequence": 1,
21902                "child_workflow_instance_id": "wf-child",
21903                "child_workflow_run_id": "run-child",
21904                "child_workflow_type": "python.child",
21905                "failure_id": "failure-child",
21906                "failure_category": "child_workflow",
21907                "message": "payment rejected",
21908                "exception": {
21909                    "type": "PaymentRejected",
21910                    "class": "payments.PaymentRejected",
21911                    "message": "payment rejected"
21912                }
21913            }),
21914        );
21915        task.workflow_type = "rust.handled-parent".to_string();
21916
21917        let commands = worker.execute_workflow_task(task).expect("handled failure");
21918        assert_eq!(commands[0]["type"], "complete_workflow");
21919        let output =
21920            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21921        assert_eq!(output["reason"], "child_workflow");
21922        assert_eq!(output["failure_id"], "failure-child");
21923        assert_eq!(output["exception_class"], "payments.PaymentRejected");
21924        assert_eq!(output["child_run_id"], "run-child");
21925    }
21926
21927    #[test]
21928    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21929        let client = Client::new("http://127.0.0.1:8080").expect("client");
21930        let mut worker = Worker::new(client, "rust-workers");
21931
21932        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21933            let signal = ctx.wait_signal("start").await?;
21934            let name = signal
21935                .first()
21936                .and_then(|value| value.as_str())
21937                .unwrap_or("world");
21938            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21939            Ok(json!({
21940                "greeting": greeting,
21941                "language": "rust"
21942            }))
21943        });
21944
21945        let signal_arguments =
21946            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21947        let task = WorkflowTask {
21948            task_id: "wft-rust-signal-1".to_string(),
21949            workflow_command_id: None,
21950            workflow_id: Some("wf-rust-hello".to_string()),
21951            run_id: Some("run-rust-hello".to_string()),
21952            workflow_type: "rust.hello_workflow".to_string(),
21953            cancel_requested: false,
21954            payload_codec: DEFAULT_CODEC.to_string(),
21955            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21956            history_events: vec![HistoryEvent {
21957                event_type: "SignalReceived".to_string(),
21958                payload: json!({
21959                    "signal_id": "sig-rust-1",
21960                    "signal_name": "start"
21961                }),
21962                raw: HashMap::new(),
21963            }],
21964            total_history_events: Some(1),
21965            history_size_bytes: None,
21966            continue_as_new_recommended: None,
21967            history_budget_pressure: None,
21968            next_history_page_token: None,
21969            workflow_task_attempt: 1,
21970            workflow_signal_id: Some("sig-rust-1".to_string()),
21971            signal_name: Some("start".to_string()),
21972            signal_arguments: Some(signal_arguments),
21973            workflow_update_id: None,
21974            update_name: None,
21975            lease_owner: Some("rust-worker".to_string()),
21976        };
21977
21978        let commands = worker.execute_workflow_task(task).expect("workflow task");
21979
21980        assert_eq!(commands.len(), 1);
21981        assert_eq!(commands[0]["type"], "schedule_activity");
21982        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21983        assert_eq!(
21984            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21985            json!(["Rust"])
21986        );
21987    }
21988
21989    #[test]
21990    fn workflow_task_appends_paginated_history_events() {
21991        let mut task = WorkflowTask {
21992            task_id: "wft-rust-pages-1".to_string(),
21993            workflow_command_id: None,
21994            workflow_id: Some("wf-rust-pages".to_string()),
21995            run_id: Some("run-rust-pages".to_string()),
21996            workflow_type: "rust.hello_workflow".to_string(),
21997            cancel_requested: false,
21998            payload_codec: DEFAULT_CODEC.to_string(),
21999            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
22000            history_events: vec![HistoryEvent {
22001                event_type: "WorkflowStarted".to_string(),
22002                payload: json!({}),
22003                raw: HashMap::new(),
22004            }],
22005            total_history_events: Some(3),
22006            history_size_bytes: None,
22007            continue_as_new_recommended: None,
22008            history_budget_pressure: None,
22009            next_history_page_token: Some("MQ==".to_string()),
22010            workflow_task_attempt: 1,
22011            workflow_signal_id: None,
22012            signal_name: None,
22013            signal_arguments: None,
22014            workflow_update_id: None,
22015            update_name: None,
22016            lease_owner: Some("rust-worker".to_string()),
22017        };
22018
22019        task.append_history_page(WorkflowTaskHistoryPage {
22020            history_events: vec![
22021                HistoryEvent {
22022                    event_type: "SignalReceived".to_string(),
22023                    payload: json!({
22024                        "signal_id": "sig-rust-1",
22025                        "signal_name": "start",
22026                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
22027                            .expect("signal arguments")
22028                    }),
22029                    raw: HashMap::new(),
22030                },
22031                HistoryEvent {
22032                    event_type: "MarkerRecorded".to_string(),
22033                    payload: json!({"sequence": 3}),
22034                    raw: HashMap::new(),
22035                },
22036            ],
22037            total_history_events: Some(3),
22038            next_history_page_token: None,
22039        });
22040
22041        assert_eq!(task.history_events.len(), 3);
22042        assert_eq!(task.total_history_events, Some(3));
22043        assert_eq!(task.next_history_page_token, None);
22044
22045        let signal = task
22046            .history_events
22047            .iter()
22048            .find(|event| event.event_type == "SignalReceived")
22049            .expect("signal event");
22050        assert_eq!(
22051            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
22052            vec![AvroValue::String("Rust".to_string())]
22053        );
22054    }
22055
22056    #[tokio::test]
22057    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
22058        let client = Client::new("http://127.0.0.1:8080").expect("client");
22059        let mut worker = Worker::new(client, "rust-workers");
22060        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22061        worker.register_query("counter", "current", |ctx, _args| async move {
22062            let mut count = 0_i64;
22063            for signal in ctx.signal_events() {
22064                let value = signal
22065                    .arguments
22066                    .first()
22067                    .and_then(Value::as_i64)
22068                    .unwrap_or_default();
22069                match signal.name.as_str() {
22070                    "increment" => count += value,
22071                    "set" => count = value,
22072                    _ => {}
22073                }
22074            }
22075            Ok(json!(count))
22076        });
22077
22078        let task = QueryTask {
22079            query_task_id: "query-rust-counter".to_string(),
22080            query_task_attempt: 1,
22081            lease_owner: Some("rust-worker".to_string()),
22082            workflow_id: Some("counter-1".to_string()),
22083            run_id: Some("run-counter-1".to_string()),
22084            workflow_type: "counter".to_string(),
22085            query_name: "current".to_string(),
22086            payload_codec: DEFAULT_CODEC.to_string(),
22087            workflow_arguments: Some(
22088                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
22089            ),
22090            query_arguments: Some(
22091                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
22092            ),
22093            history_events: vec![
22094                HistoryEvent {
22095                    event_type: "SignalReceived".to_string(),
22096                    payload: json!({
22097                        "signal_id": "php-signal-1",
22098                        "signal_name": "increment",
22099                        "workflow_sequence": 1,
22100                        "payload_codec": DEFAULT_CODEC,
22101                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
22102                    }),
22103                    raw: HashMap::new(),
22104                },
22105                HistoryEvent {
22106                    event_type: "SignalReceived".to_string(),
22107                    payload: json!({
22108                        "signal_id": "python-signal-2",
22109                        "signal_name": "increment",
22110                        "workflow_sequence": 2,
22111                        "payload_codec": DEFAULT_CODEC,
22112                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
22113                    }),
22114                    raw: HashMap::new(),
22115                },
22116                HistoryEvent {
22117                    event_type: "SignalReceived".to_string(),
22118                    payload: json!({
22119                        "signal_id": "rust-signal-3",
22120                        "signal_name": "set",
22121                        "workflow_sequence": 3,
22122                        "payload_codec": DEFAULT_CODEC,
22123                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
22124                    }),
22125                    raw: HashMap::new(),
22126                },
22127            ],
22128            history_export: None,
22129            run_status: Some("completed".to_string()),
22130        };
22131
22132        let result = worker.execute_query_task(task).await.expect("query result");
22133        assert_eq!(result.into_json().expect("query projection"), json!(0));
22134    }
22135
22136    #[tokio::test]
22137    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
22138        let worker = replay_counter_worker();
22139        let running_history = json!([
22140            {
22141                "type": "ActivityCompleted",
22142                "payload": {
22143                    "sequence": 1,
22144                    "activity_type": "load-counter",
22145                    "payload_codec": DEFAULT_CODEC,
22146                    "result": fixture_envelope(json!("loaded"))
22147                }
22148            },
22149            {
22150                "type": "SignalWaitOpened",
22151                "payload": {
22152                    "sequence": 3,
22153                    "signal_name": "increment"
22154                }
22155            },
22156            {
22157                "type": "SignalReceived",
22158                "payload": {
22159                    "signal_id": "signal-3",
22160                    "signal_name": "increment",
22161                    "workflow_sequence": 2,
22162                    "payload_codec": DEFAULT_CODEC,
22163                    "arguments": fixture_envelope(json!([3]))
22164                }
22165            },
22166            {
22167                "type": "SignalApplied",
22168                "payload": {
22169                    "sequence": 3,
22170                    "signal_id": "signal-3",
22171                    "signal_name": "increment",
22172                    "payload_codec": DEFAULT_CODEC,
22173                    "value": fixture_envelope(json!([3]))
22174                }
22175            }
22176        ]);
22177
22178        let running = worker
22179            .execute_query_task(replay_counter_query(
22180                "current",
22181                running_history.clone(),
22182                "running",
22183            ))
22184            .await
22185            .expect("running replay query");
22186        assert_eq!(
22187            running.clone().into_json().expect("query projection"),
22188            json!({"loaded": "loaded", "count": 3, "finished": false})
22189        );
22190
22191        let detached = worker
22192            .execute_query_task(replay_counter_query(
22193                "detached-mutation",
22194                running_history.clone(),
22195                "running",
22196            ))
22197            .await
22198            .expect("query mutates only its detached state clone");
22199        assert_eq!(detached.into_json().expect("query projection"), json!(999));
22200        let failed = worker
22201            .execute_query_task(replay_counter_query(
22202                "failed-mutation",
22203                running_history.clone(),
22204                "running",
22205            ))
22206            .await
22207            .expect_err("failed query");
22208        assert_eq!(failed.reason, "query_rejected");
22209        let unchanged = worker
22210            .execute_query_task(replay_counter_query("current", running_history, "running"))
22211            .await
22212            .expect("later query reconstructs unchanged state");
22213        assert_eq!(unchanged, running);
22214
22215        let restarted_worker = replay_counter_worker();
22216        let empty_arguments = fixture_envelope(json!([]));
22217        let loaded_result = fixture_envelope(json!("loaded"));
22218        let signal_three = fixture_blob(json!([3]));
22219        let signal_five = fixture_blob(json!([5]));
22220        let restarted_task: QueryTask = serde_json::from_value(json!({
22221            "query_task_id": "query-after-restart",
22222            "workflow_id": "counter-1",
22223            "run_id": "run-counter-1",
22224            "workflow_type": "replay-counter",
22225            "query_name": "current",
22226            "payload_codec": DEFAULT_CODEC,
22227            "workflow_arguments": empty_arguments.clone(),
22228            "query_arguments": empty_arguments,
22229            "history_events": [],
22230            "history_export": {
22231                "payloads": {"codec": DEFAULT_CODEC},
22232                "history_events": [
22233                    {
22234                        "type": "ActivityCompleted",
22235                        "payload": {
22236                            "sequence": 1,
22237                            "activity_type": "load-counter",
22238                            "payload_codec": DEFAULT_CODEC,
22239                            "result": null
22240                        }
22241                    },
22242                    {
22243                        "type": "SignalWaitOpened",
22244                        "payload": {
22245                            "sequence": 3,
22246                            "signal_name": "increment"
22247                        }
22248                    },
22249                    {
22250                        "type": "SignalReceived",
22251                        "payload": {
22252                            "signal_id": "signal-3",
22253                            "signal_name": "increment",
22254                            "workflow_sequence": 2
22255                        }
22256                    },
22257                    {
22258                        "type": "SignalApplied",
22259                        "payload": {
22260                            "sequence": 3,
22261                            "signal_id": "signal-3",
22262                            "signal_name": "increment"
22263                        }
22264                    },
22265                    {
22266                        "type": "SignalWaitOpened",
22267                        "payload": {
22268                            "sequence": 5,
22269                            "signal_name": "increment"
22270                        }
22271                    },
22272                    {
22273                        "type": "SignalReceived",
22274                        "payload": {
22275                            "signal_id": "signal-5",
22276                            "signal_name": "increment",
22277                            "workflow_sequence": 4
22278                        }
22279                    },
22280                    {
22281                        "type": "SignalApplied",
22282                        "payload": {
22283                            "sequence": 5,
22284                            "signal_id": "signal-5",
22285                            "signal_name": "increment"
22286                        }
22287                    }
22288                ],
22289                "activities": [{
22290                    "sequence": 1,
22291                    "activity_type": "load-counter",
22292                    "payload_codec": DEFAULT_CODEC,
22293                    "result": loaded_result
22294                }],
22295                "signals": [
22296                    {
22297                        "id": "signal-3",
22298                        "name": "increment",
22299                        "workflow_sequence": 2,
22300                        "payload_codec": DEFAULT_CODEC,
22301                        "arguments": signal_three
22302                    },
22303                    {
22304                        "id": "signal-5",
22305                        "name": "increment",
22306                        "workflow_sequence": 4,
22307                        "payload_codec": DEFAULT_CODEC,
22308                        "arguments": signal_five
22309                    }
22310                ]
22311            },
22312            "run_status": "completed"
22313        }))
22314        .expect("cold replay query task");
22315        let completed = restarted_worker
22316            .execute_query_task(restarted_task)
22317            .await
22318            .expect("completed cold replay query");
22319        assert_eq!(
22320            completed.into_json().expect("query projection"),
22321            json!({"loaded": "loaded", "count": 8, "finished": true})
22322        );
22323    }
22324
22325    #[tokio::test]
22326    async fn replayed_query_replay_failures_are_machine_readable() {
22327        let worker = replay_counter_worker();
22328        let task = replay_counter_query(
22329            "current",
22330            json!([{
22331                "type": "ActivityCompleted",
22332                "payload": {
22333                    "sequence": 1,
22334                    "payload_codec": DEFAULT_CODEC,
22335                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
22336                }
22337            }]),
22338            "running",
22339        );
22340        let failure = worker
22341            .execute_query_task(task)
22342            .await
22343            .expect_err("invalid replay history payload");
22344        assert_eq!(failure.reason, "query_payload_decode_failed");
22345        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
22346        assert!(failure.message.contains("invalid_payload_framing"));
22347    }
22348
22349    #[tokio::test]
22350    async fn query_task_restores_compact_history_from_export() {
22351        let client = Client::new("http://127.0.0.1:8080").expect("client");
22352        let mut worker = Worker::new(client, "rust-workers");
22353        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22354        worker.register_query("counter", "current", |ctx, _args| async move {
22355            Ok(json!(ctx.signals("increment")[0][0]))
22356        });
22357        let empty_arguments = fixture_envelope(json!([]));
22358        let exported_signal = fixture_blob(json!([9]));
22359        let task: QueryTask = serde_json::from_value(json!({
22360            "query_task_id": "query-export",
22361            "workflow_type": "counter",
22362            "query_name": "current",
22363            "payload_codec": DEFAULT_CODEC,
22364            "workflow_arguments": empty_arguments.clone(),
22365            "query_arguments": empty_arguments,
22366            "history_events": [],
22367            "history_export": {
22368                "payloads": {"codec": DEFAULT_CODEC},
22369                "history_events": [{
22370                    "type": "SignalReceived",
22371                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
22372                }],
22373                "signals": [{
22374                    "id": "signal-export",
22375                    "name": "increment",
22376                    "status": "applied",
22377                    "workflow_sequence": 1,
22378                    "payload_codec": DEFAULT_CODEC,
22379                    "arguments": exported_signal
22380                }]
22381            }
22382        }))
22383        .expect("query task");
22384
22385        let result = worker.execute_query_task(task).await.expect("query result");
22386        assert_eq!(result.into_json().expect("query projection"), json!(9));
22387    }
22388
22389    #[tokio::test]
22390    async fn query_task_failures_have_stable_reasons() {
22391        let client = Client::new("http://127.0.0.1:8080").expect("client");
22392        let mut worker = Worker::new(client, "rust-workers");
22393        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22394        worker.register_query(
22395            "counter",
22396            "current",
22397            |_ctx, _args| async move { Ok(json!(0)) },
22398        );
22399
22400        let base_task = QueryTask {
22401            query_task_id: "query-errors".to_string(),
22402            query_task_attempt: 1,
22403            lease_owner: None,
22404            workflow_id: Some("counter-errors".to_string()),
22405            run_id: Some("run-errors".to_string()),
22406            workflow_type: "counter".to_string(),
22407            query_name: "missing".to_string(),
22408            payload_codec: DEFAULT_CODEC.to_string(),
22409            workflow_arguments: Some(fixture_envelope(json!([]))),
22410            query_arguments: Some(fixture_envelope(json!([]))),
22411            history_events: Vec::new(),
22412            history_export: None,
22413            run_status: Some("running".to_string()),
22414        };
22415
22416        let unknown = worker
22417            .execute_query_task(base_task.clone())
22418            .await
22419            .expect_err("unknown query");
22420        assert_eq!(unknown.reason, "rejected_unknown_query");
22421
22422        let mut malformed = base_task;
22423        malformed.query_name = "current".to_string();
22424        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
22425        let malformed = worker
22426            .execute_query_task(malformed)
22427            .await
22428            .expect_err("malformed payload");
22429        assert_eq!(malformed.reason, "query_payload_decode_failed");
22430
22431        let client = Client::new("http://127.0.0.1:8080").expect("client");
22432        let mut unavailable_worker = Worker::new(client, "rust-workers");
22433        unavailable_worker
22434            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22435        let empty_arguments = fixture_envelope(json!([]));
22436        let unavailable_task: QueryTask = serde_json::from_value(json!({
22437            "query_task_id": "query-unavailable",
22438            "workflow_type": "counter",
22439            "query_name": "current",
22440            "payload_codec": DEFAULT_CODEC,
22441            "workflow_arguments": empty_arguments.clone(),
22442            "query_arguments": empty_arguments
22443        }))
22444        .expect("query task");
22445        let unavailable = unavailable_worker
22446            .execute_query_task(unavailable_task)
22447            .await
22448            .expect_err("query handler unavailable");
22449        assert_eq!(unavailable.reason, "query_handler_unavailable");
22450    }
22451
22452    #[tokio::test]
22453    async fn client_query_decodes_result_and_typed_failure() {
22454        let server = MockWorkerServer::start();
22455        let client = Client::builder(server.base_url())
22456            .timeout(Duration::from_secs(2))
22457            .build()
22458            .expect("client");
22459
22460        let result = client
22461            .query_workflow("counter-1", "current", json!([]))
22462            .await
22463            .expect("query result");
22464        assert_eq!(result, json!({"count": 8}));
22465
22466        let error = client
22467            .query_workflow("counter-1", "missing", json!([]))
22468            .await
22469            .expect_err("unknown query");
22470        let Error::QueryFailed(failure) = error else {
22471            panic!("expected typed query failure");
22472        };
22473        assert_eq!(failure.status, 404);
22474        assert_eq!(failure.reason, "rejected_unknown_query");
22475    }
22476
22477    #[tokio::test]
22478    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
22479        let server = MockWorkerServer::start();
22480        let client = Client::builder(server.base_url())
22481            .timeout(Duration::from_secs(2))
22482            .build()
22483            .expect("client");
22484        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
22485
22486        client
22487            .start_workflow(
22488                "typed.echo",
22489                "rust-workers",
22490                "typed-start",
22491                arguments.clone(),
22492            )
22493            .await
22494            .expect("typed workflow start");
22495        assert_eq!(
22496            decode_wire_avro_value(
22497                &server.request_body("/api/workflows")["input"],
22498                DEFAULT_CODEC,
22499            )
22500            .expect("typed start input"),
22501            arguments
22502        );
22503
22504        client
22505            .signal_workflow("typed-1", "changed", arguments.clone())
22506            .await
22507            .expect("typed signal");
22508        assert_eq!(
22509            decode_wire_avro_value(
22510                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
22511                DEFAULT_CODEC,
22512            )
22513            .expect("typed signal input"),
22514            arguments
22515        );
22516
22517        assert_eq!(
22518            client
22519                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22520                .await
22521                .expect("typed query"),
22522            typed_fidelity_probe()
22523        );
22524        assert_eq!(
22525            decode_wire_avro_value(
22526                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22527                DEFAULT_CODEC,
22528            )
22529            .expect("typed query input"),
22530            arguments
22531        );
22532
22533        assert_eq!(
22534            client
22535                .update_workflow_avro_value(
22536                    "typed-1",
22537                    "replace",
22538                    arguments.clone(),
22539                    Some("typed-request"),
22540                )
22541                .await
22542                .expect("typed update"),
22543            typed_fidelity_probe()
22544        );
22545        let update = server.request_body("/api/workflows/typed-1/update/replace");
22546        assert_eq!(update["request_id"], "typed-request");
22547        assert_eq!(
22548            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22549            arguments
22550        );
22551
22552        let handle = WorkflowHandle {
22553            client: client.clone(),
22554            workflow_id: "typed-1".to_string(),
22555            run_id: Some("run-typed-1".to_string()),
22556            workflow_type: "typed.echo".to_string(),
22557        };
22558        assert_eq!(
22559            handle
22560                .result_avro_value(WorkflowResultOptions::default())
22561                .await
22562                .expect("typed workflow result"),
22563            typed_fidelity_probe()
22564        );
22565
22566        client
22567            .complete_activity_task(
22568                "activity-typed",
22569                "attempt-typed",
22570                "rust-worker",
22571                typed_fidelity_probe(),
22572                DEFAULT_CODEC,
22573            )
22574            .await
22575            .expect("typed activity completion");
22576        assert_eq!(
22577            decode_wire_avro_value(
22578                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22579                    ["result"],
22580                DEFAULT_CODEC,
22581            )
22582            .expect("typed activity result"),
22583            typed_fidelity_probe()
22584        );
22585        client
22586            .fail_activity_task(
22587                "activity-typed",
22588                "attempt-typed",
22589                "rust-worker",
22590                "typed failure",
22591                true,
22592            )
22593            .await
22594            .expect("activity failure");
22595    }
22596
22597    #[tokio::test]
22598    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22599        let server = MockWorkerServer::start();
22600        let client = Client::builder(server.base_url())
22601            .timeout(Duration::from_secs(2))
22602            .build()
22603            .expect("client");
22604
22605        let options = WorkflowCommandOptions::new()
22606            .reason("cleanup requested")
22607            .request_id("cancel-17");
22608        let cancelled = client
22609            .cancel_workflow("wf-lifecycle", options)
22610            .await
22611            .expect("instance cancellation");
22612        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22613        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22614        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22615        assert_eq!(
22616            server.request_body("/api/workflows/wf-lifecycle/cancel"),
22617            json!({"reason":"cleanup requested","request_id":"cancel-17"})
22618        );
22619
22620        let terminated = client
22621            .terminate_workflow(
22622                "wf-lifecycle",
22623                WorkflowCommandOptions::new().reason("forced stop"),
22624            )
22625            .await
22626            .expect("instance termination");
22627        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22628        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22629
22630        client
22631            .cancel_workflow_run(
22632                "wf-lifecycle",
22633                "run-current",
22634                WorkflowCommandOptions::default(),
22635            )
22636            .await
22637            .expect("selected run cancellation");
22638        client
22639            .terminate_workflow_run(
22640                "wf-lifecycle",
22641                "run-current",
22642                WorkflowCommandOptions::default(),
22643            )
22644            .await
22645            .expect("selected run termination");
22646
22647        for (command, error) in [
22648            (
22649                WorkflowCommandKind::Cancel,
22650                client
22651                    .cancel_workflow_run(
22652                        "wf-lifecycle",
22653                        "run-stale",
22654                        WorkflowCommandOptions::default(),
22655                    )
22656                    .await
22657                    .expect_err("stale cancellation must be rejected"),
22658            ),
22659            (
22660                WorkflowCommandKind::Terminate,
22661                client
22662                    .terminate_workflow_run(
22663                        "wf-lifecycle",
22664                        "run-stale",
22665                        WorkflowCommandOptions::default(),
22666                    )
22667                    .await
22668                    .expect_err("stale termination must be rejected"),
22669            ),
22670        ] {
22671            let Error::WorkflowCommandRejected(rejection) = error else {
22672                panic!("expected typed command rejection");
22673            };
22674            assert_eq!(rejection.command, command);
22675            assert_eq!(rejection.status, 409);
22676            assert_eq!(rejection.reason, "historical_run_command_rejected");
22677            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22678            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22679        }
22680    }
22681
22682    #[tokio::test]
22683    async fn redrive_targets_a_failed_run_and_returns_successor_identity() {
22684        let server = MockWorkerServer::start();
22685        let client = Client::builder(server.base_url())
22686            .timeout(Duration::from_secs(2))
22687            .build()
22688            .expect("client");
22689
22690        let result = client
22691            .redrive_workflow_run("wf-lifecycle", "run-failed", Some("retry-1"))
22692            .await
22693            .expect("redrive accepted");
22694        assert_eq!(result.workflow_id, "wf-lifecycle");
22695        assert_eq!(result.source_run_id, "run-failed");
22696        assert_eq!(result.run_id, "run-successor");
22697        assert_eq!(result.resume_step_sequence, 2);
22698        assert_eq!(
22699            server.request_body("/api/workflows/wf-lifecycle/runs/run-failed/redrive"),
22700            json!({"request_id":"retry-1"})
22701        );
22702
22703        let repeated = client
22704            .redrive_workflow_run("wf-lifecycle", "run-failed-existing", Some("retry-1"))
22705            .await
22706            .expect("idempotent redrive response accepted");
22707        assert_eq!(repeated.run_id, "run-successor");
22708
22709        let error = client
22710            .redrive_workflow_run("wf-lifecycle", "run-completed", None)
22711            .await
22712            .expect_err("completed run must be rejected");
22713        let Error::Http { status, body } = error else {
22714            panic!("expected HTTP redrive rejection");
22715        };
22716        assert_eq!(status.as_u16(), 409);
22717        assert!(body.contains("run_not_failed"));
22718        assert_eq!(
22719            server.request_body("/api/workflows/wf-lifecycle/runs/run-completed/redrive"),
22720            json!({})
22721        );
22722
22723        let handle = WorkflowHandle {
22724            client: client.clone(),
22725            workflow_id: "wf-lifecycle".to_string(),
22726            run_id: Some("run-failed".to_string()),
22727            workflow_type: "test".to_string(),
22728        };
22729        assert_eq!(
22730            handle.redrive(None).await.expect("handle redrive").run_id,
22731            "run-successor"
22732        );
22733        let missing_run = WorkflowHandle {
22734            run_id: None,
22735            ..handle
22736        };
22737        assert!(matches!(
22738            missing_run.redrive(None).await,
22739            Err(Error::Codec(_))
22740        ));
22741    }
22742
22743    #[tokio::test]
22744    async fn workflow_start_options_send_server_enforced_deadlines() {
22745        let server = MockWorkerServer::start();
22746        let client = Client::builder(server.base_url())
22747            .timeout(Duration::from_secs(2))
22748            .build()
22749            .expect("client");
22750
22751        let handle = client
22752            .start_workflow_with_options(
22753                "rust.timeout",
22754                "rust-timeouts",
22755                "wf-start-options",
22756                WorkflowStartOptions::new()
22757                    .execution_timeout_seconds(30)
22758                    .run_timeout_seconds(1),
22759                json!([]),
22760            )
22761            .await
22762            .expect("workflow start");
22763
22764        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22765        let body = server.request_body("/api/workflows");
22766        assert_eq!(body["execution_timeout_seconds"], 30);
22767        assert_eq!(body["run_timeout_seconds"], 1);
22768
22769        let invalid = client
22770            .start_workflow_with_options(
22771                "rust.timeout",
22772                "rust-timeouts",
22773                "wf-invalid-options",
22774                WorkflowStartOptions::new()
22775                    .execution_timeout_seconds(1)
22776                    .run_timeout_seconds(2),
22777                json!([]),
22778            )
22779            .await
22780            .expect_err("invalid deadline ordering");
22781        assert!(invalid
22782            .to_string()
22783            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22784    }
22785
22786    #[tokio::test]
22787    async fn workflow_result_returns_each_typed_terminal_outcome() {
22788        let server = MockWorkerServer::start();
22789        let client = Client::builder(server.base_url())
22790            .timeout(Duration::from_secs(2))
22791            .build()
22792            .expect("client");
22793        let options = WorkflowResultOptions {
22794            poll_interval: Duration::ZERO,
22795            timeout: Duration::from_secs(1),
22796        };
22797
22798        let failed = WorkflowHandle {
22799            client: client.clone(),
22800            workflow_id: "wf-failed".to_string(),
22801            run_id: Some("run-failed".to_string()),
22802            workflow_type: "failure".to_string(),
22803        }
22804        .result(options)
22805        .await
22806        .expect_err("failed outcome");
22807        let Error::WorkflowFailed(failure) = failed else {
22808            panic!("expected WorkflowFailed");
22809        };
22810        assert_eq!(failure.workflow_id, "wf-failed");
22811        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22812        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22813        assert_eq!(failure.failure_category.as_deref(), Some("application"));
22814        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22815        assert_eq!(
22816            failure.exception_class.as_deref(),
22817            Some("billing::PaymentError")
22818        );
22819        assert_eq!(failure.non_retryable, Some(true));
22820
22821        for (workflow_id, expected_kind, expected_reason) in [
22822            (
22823                "wf-cancelled",
22824                WorkflowTerminalKind::Cancelled,
22825                "cleanup requested",
22826            ),
22827            (
22828                "wf-terminated",
22829                WorkflowTerminalKind::Terminated,
22830                "forced stop",
22831            ),
22832            (
22833                "wf-timed-out",
22834                WorkflowTerminalKind::TimedOut,
22835                "run_timeout",
22836            ),
22837        ] {
22838            let error = WorkflowHandle {
22839                client: client.clone(),
22840                workflow_id: workflow_id.to_string(),
22841                run_id: None,
22842                workflow_type: "terminal".to_string(),
22843            }
22844            .result(options)
22845            .await
22846            .expect_err("typed terminal outcome");
22847            let outcome = match error {
22848                Error::WorkflowCancelled(outcome) => outcome,
22849                Error::WorkflowTerminated(outcome) => outcome,
22850                Error::WorkflowTimedOut(outcome) => outcome,
22851                other => panic!("unexpected terminal error: {other}"),
22852            };
22853            assert_eq!(outcome.kind, expected_kind);
22854            assert_eq!(outcome.workflow_id, workflow_id);
22855            assert_eq!(outcome.reason, expected_reason);
22856        }
22857
22858        let wait_timeout = WorkflowHandle {
22859            client,
22860            workflow_id: "wf-waiting".to_string(),
22861            run_id: Some("run-waiting".to_string()),
22862            workflow_type: "waiting".to_string(),
22863        }
22864        .result(WorkflowResultOptions {
22865            poll_interval: Duration::ZERO,
22866            timeout: Duration::ZERO,
22867        })
22868        .await
22869        .expect_err("client wait timeout");
22870        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22871            panic!("expected typed client timeout");
22872        };
22873        assert_eq!(timeout.reason, "result_wait_timeout");
22874        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22875        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22876    }
22877
22878    #[tokio::test]
22879    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22880        let server = MockWorkerServer::start();
22881        let client = Client::builder(server.base_url())
22882            .timeout(Duration::from_secs(2))
22883            .build()
22884            .expect("client");
22885
22886        let handle = WorkflowHandle {
22887            client,
22888            workflow_id: "wf-selected".to_string(),
22889            run_id: Some("run-selected".to_string()),
22890            workflow_type: "selected".to_string(),
22891        };
22892        let options = WorkflowResultOptions {
22893            poll_interval: Duration::ZERO,
22894            timeout: Duration::from_secs(1),
22895        };
22896
22897        let current = handle
22898            .result(options)
22899            .await
22900            .expect("instance result follows the current run");
22901        assert_eq!(current, json!("current run output"));
22902
22903        let error = handle
22904            .result_selected_run(options)
22905            .await
22906            .expect_err("the selected run is cancelled even though the current run completed");
22907
22908        let Error::WorkflowCancelled(outcome) = error else {
22909            panic!("expected selected run cancellation");
22910        };
22911        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22912        assert_eq!(outcome.reason, "selected run cancelled");
22913        assert_eq!(
22914            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22915            1
22916        );
22917        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22918    }
22919
22920    #[tokio::test]
22921    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22922        let server = MockWorkerServer::draining_polls();
22923        let client = Client::builder(server.base_url())
22924            .timeout(Duration::from_secs(2))
22925            .build()
22926            .expect("client");
22927
22928        let workflow = client
22929            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22930            .await
22931            .expect("workflow drain response");
22932        let activity = client
22933            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22934            .await
22935            .expect("activity drain response");
22936        let query = client
22937            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22938            .await
22939            .expect("query drain response");
22940
22941        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22942            assert_eq!(
22943                outcome,
22944                WorkerPollOutcome::Stop {
22945                    poll_status: Some("draining".to_string()),
22946                    reason: Some("worker_draining".to_string()),
22947                }
22948            );
22949        }
22950
22951        assert!(client
22952            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22953            .await
22954            .expect("compatibility poll")
22955            .is_none());
22956    }
22957
22958    #[tokio::test]
22959    async fn managed_worker_honors_drain_stop_for_every_task_family() {
22960        let server = MockWorkerServer::draining_polls();
22961        let client = Client::builder(server.base_url())
22962            .timeout(Duration::from_secs(2))
22963            .build()
22964            .expect("client");
22965
22966        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22967            .worker_id("draining-workflow-worker")
22968            .poll_timeout(Duration::ZERO);
22969        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22970        workflow_worker
22971            .run()
22972            .await
22973            .expect("workflow drain is a clean stop");
22974
22975        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22976            .worker_id("draining-activity-worker")
22977            .poll_timeout(Duration::ZERO);
22978        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22979        activity_worker
22980            .run()
22981            .await
22982            .expect("activity drain is a clean stop");
22983
22984        let mut query_worker = Worker::new(client, "rust-workers")
22985            .worker_id("draining-query-worker")
22986            .poll_timeout(Duration::ZERO);
22987        query_worker.register_query("counter", "current", |_ctx, _args| async {
22988            Ok(Value::Null)
22989        });
22990        query_worker
22991            .run()
22992            .await
22993            .expect("query drain is a clean stop");
22994    }
22995
22996    #[tokio::test]
22997    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22998        let server = MockWorkerServer::start();
22999        let client = Client::builder(server.base_url())
23000            .timeout(Duration::from_secs(2))
23001            .build()
23002            .expect("client");
23003
23004        let heartbeat = client
23005            .heartbeat_activity_task(
23006                "activity-cancel",
23007                "attempt-cancel",
23008                "rust-worker",
23009                typed_fidelity_probe(),
23010            )
23011            .await
23012            .expect("cancellation heartbeat");
23013        assert!(heartbeat.cancel_requested);
23014        assert!(heartbeat.should_stop());
23015        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
23016        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
23017        let heartbeat_body =
23018            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
23019        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
23020        assert_eq!(
23021            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
23022                .expect("typed heartbeat details"),
23023            typed_fidelity_probe()
23024        );
23025
23026        let error = client
23027            .complete_activity_task(
23028                "activity-cancel",
23029                "attempt-cancel",
23030                "rust-worker",
23031                json!({"late":true}),
23032                DEFAULT_CODEC,
23033            )
23034            .await
23035            .expect_err("late completion must be refused");
23036        assert!(activity_task_rejection_is_final(&error));
23037        let Error::ActivityTaskRejected(rejection) = error else {
23038            panic!("expected typed activity rejection");
23039        };
23040        assert_eq!(rejection.status, 409);
23041        assert_eq!(rejection.reason, "run_cancelled");
23042        assert!(rejection.cancel_requested);
23043        assert_eq!(rejection.can_continue, Some(false));
23044    }
23045
23046    #[tokio::test]
23047    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
23048        let server = MockWorkerServer::cancelled_activity();
23049        let client = Client::builder(server.base_url())
23050            .timeout(Duration::from_secs(2))
23051            .build()
23052            .expect("client");
23053        let cancellation_observed = Arc::new(AtomicBool::new(false));
23054        let observed = Arc::clone(&cancellation_observed);
23055        let mut worker = Worker::new(client.clone(), "rust-workers")
23056            .worker_id("rust-cancel-worker")
23057            .poll_timeout(Duration::from_millis(10));
23058        worker.register_activity("cancel-aware", move |ctx, _args| {
23059            let observed = Arc::clone(&observed);
23060            async move {
23061                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
23062                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
23063                Ok(json!({"late":"completion"}))
23064            }
23065        });
23066
23067        assert_eq!(
23068            worker.run_once().await.expect("cancelled attempt handled"),
23069            1
23070        );
23071        assert!(cancellation_observed.load(Ordering::SeqCst));
23072        assert_eq!(
23073            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
23074            1
23075        );
23076
23077        let mut restarted = Worker::new(client, "rust-workers")
23078            .worker_id("rust-cancel-worker-restarted")
23079            .poll_timeout(Duration::from_millis(10));
23080        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
23081        assert_eq!(
23082            restarted
23083                .run_once()
23084                .await
23085                .expect("replacement worker continues polling"),
23086            0
23087        );
23088    }
23089
23090    #[tokio::test]
23091    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
23092        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
23093        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
23094        let client = Client::builder(server.base_url())
23095            .timeout(Duration::from_secs(2))
23096            .build()
23097            .expect("client");
23098
23099        let direct_error = client
23100            .complete_workflow_task(
23101                "workflow-timeout-task",
23102                "timeout-worker",
23103                3,
23104                vec![json!({
23105                    "type": "complete_workflow",
23106                    "result": fixture_envelope(Value::Null)
23107                })],
23108            )
23109            .await
23110            .expect_err("the low-level client preserves the completion rejection");
23111        let Error::Http { status, body } = direct_error else {
23112            panic!("expected the original HTTP completion rejection");
23113        };
23114        assert_eq!(status, reqwest::StatusCode::CONFLICT);
23115        assert_eq!(
23116            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
23117            "run_timed_out"
23118        );
23119
23120        let mut worker = Worker::new(client, "rust-workers")
23121            .worker_id("timeout-worker")
23122            .poll_timeout(Duration::from_millis(10));
23123        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
23124            Ok(json!({"late": "result"}))
23125        });
23126
23127        assert_eq!(
23128            worker
23129                .run_once()
23130                .await
23131                .expect("authoritative selected-run timeout settles the tick"),
23132            1
23133        );
23134        assert_eq!(
23135            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
23136            2,
23137            "both the direct client proof and managed worker must see the rejection"
23138        );
23139    }
23140
23141    #[tokio::test]
23142    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
23143        for (name, status, response) in [
23144            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
23145            (
23146                "command was recorded",
23147                "409 Conflict",
23148                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23149            ),
23150            (
23151                "lease conflict",
23152                "409 Conflict",
23153                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
23154            ),
23155            (
23156                "nonterminal run",
23157                "409 Conflict",
23158                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
23159            ),
23160            (
23161                "different selected run",
23162                "409 Conflict",
23163                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
23164            ),
23165            (
23166                "different task attempt",
23167                "409 Conflict",
23168                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23169            ),
23170            (
23171                "authentication failure",
23172                "401 Unauthorized",
23173                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23174            ),
23175            (
23176                "authorization failure",
23177                "403 Forbidden",
23178                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23179            ),
23180            (
23181                "protocol failure",
23182                "400 Bad Request",
23183                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
23184            ),
23185            (
23186                "malformed command",
23187                "422 Unprocessable Entity",
23188                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23189            ),
23190            (
23191                "transient server failure",
23192                "503 Service Unavailable",
23193                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23194            ),
23195        ] {
23196            let server = MockWorkerServer::workflow_completion(status, response);
23197            let client = Client::builder(server.base_url())
23198                .timeout(Duration::from_secs(2))
23199                .build()
23200                .expect("client");
23201            let mut worker = Worker::new(client, "rust-workers")
23202                .worker_id("timeout-worker")
23203                .poll_timeout(Duration::from_millis(10));
23204            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
23205                Ok(json!({"late": "result"}))
23206            });
23207
23208            let error = worker
23209                .run_once()
23210                .await
23211                .expect_err(&format!("{name} must remain an error"));
23212            assert!(
23213                matches!(error, Error::Http { .. } | Error::Protocol(_)),
23214                "{name} returned an unexpected error variant: {error}"
23215            );
23216        }
23217    }
23218
23219    #[tokio::test]
23220    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
23221        let server = MockWorkerServer::start();
23222        let client = Client::builder(server.base_url())
23223            .worker_token(Some("worker-secret".to_string()))
23224            .namespace("orders")
23225            .timeout(Duration::from_secs(2))
23226            .build()
23227            .expect("client");
23228        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
23229
23230        let result = client
23231            .deregister_worker_registration("worker/α space")
23232            .await
23233            .expect("deregister worker registration");
23234
23235        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
23236        assert_eq!(
23237            server.worker_protocol_for(path).as_deref(),
23238            Some(WORKER_PROTOCOL_VERSION)
23239        );
23240        assert_eq!(server.control_protocol_for(path), None);
23241        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
23242        assert_eq!(
23243            server.authorization_for(path).as_deref(),
23244            Some("Bearer worker-secret")
23245        );
23246        assert_eq!(
23247            result,
23248            WorkerDeregistrationEnvelope {
23249                worker_id: "deregistered-worker".to_string(),
23250                outcome: "deregistered".to_string(),
23251                recovered_workflow_task_count: 2,
23252            }
23253        );
23254    }
23255
23256    #[tokio::test]
23257    async fn low_level_registration_rejects_update_validators_before_transport() {
23258        let server = MockWorkerServer::start();
23259        let client = Client::builder(server.base_url())
23260            .timeout(Duration::from_secs(2))
23261            .build()
23262            .expect("client");
23263
23264        for update_validators in [json!(["approve"]), json!("approve")] {
23265            let error = client
23266                .register_worker_with_command_contracts(
23267                    "validator-claiming-worker",
23268                    "rust-workers",
23269                    vec!["orders".to_string()],
23270                    vec![],
23271                    1,
23272                    1,
23273                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
23274                    json!({
23275                        "orders": {
23276                            "queries": ["current"],
23277                            "updates": ["approve"],
23278                            "update_validators": update_validators,
23279                        },
23280                    }),
23281                )
23282                .await
23283                .expect_err("unsupported validator claims must fail before registration");
23284
23285            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
23286                panic!("expected typed unsupported-validator failure");
23287            };
23288            assert_eq!(workflow_type, "orders");
23289        }
23290        assert_eq!(server.request_count("/api/worker/register"), 0);
23291    }
23292
23293    #[tokio::test]
23294    async fn low_level_registration_preserves_query_and_update_contracts() {
23295        let server = MockWorkerServer::start();
23296        let client = Client::builder(server.base_url())
23297            .timeout(Duration::from_secs(2))
23298            .build()
23299            .expect("client");
23300        let contracts = json!({
23301            "orders": {
23302                "queries": ["current"],
23303                "updates": ["approve"],
23304                "update_validators": [],
23305            },
23306            "payments": {
23307                "queries": ["status"],
23308                "updates": ["capture"],
23309            },
23310        });
23311
23312        client
23313            .register_worker_with_command_contracts(
23314                "command-worker",
23315                "rust-workers",
23316                vec!["orders".to_string(), "payments".to_string()],
23317                vec![],
23318                1,
23319                1,
23320                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
23321                contracts.clone(),
23322            )
23323            .await
23324            .expect("query and update contracts must remain supported");
23325
23326        assert_eq!(
23327            server.request_body("/api/worker/register")["workflow_command_contracts"],
23328            contracts
23329        );
23330        assert!(server
23331            .request_body("/api/worker/register")
23332            .get("workflow_definition_fingerprints")
23333            .is_none());
23334    }
23335
23336    #[tokio::test]
23337    async fn rust_worker_advertises_source_backed_definition_and_clears_it_on_reregistration() {
23338        let server = MockWorkerServer::start();
23339        let client = Client::builder(server.base_url())
23340            .timeout(Duration::from_secs(2))
23341            .build()
23342            .expect("client");
23343        let mut worker = Worker::new(client, "rust-workers");
23344        worker.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
23345
23346        assert!(worker
23347            .set_workflow_definition_sources("missing", &[include_str!("lib.rs")])
23348            .is_err());
23349        assert!(worker
23350            .set_workflow_definition_sources("orders", &[])
23351            .is_err());
23352        worker
23353            .set_workflow_definition_sources("orders", &[include_str!("lib.rs")])
23354            .expect("embed workflow source");
23355        worker.register().await.expect("register worker");
23356        let first = server.request_bodies("/api/worker/register")[0]
23357            ["workflow_definition_fingerprints"]["orders"]
23358            .as_str()
23359            .expect("source-backed fingerprint")
23360            .to_string();
23361        assert!(first.starts_with("sha256:"));
23362
23363        worker
23364            .set_workflow_definition_sources("orders", &[include_str!("lib.rs"), "changed helper"])
23365            .expect("change embedded helper source");
23366        worker.register().await.expect("register changed source");
23367        let changed = server.request_bodies("/api/worker/register")[1]
23368            ["workflow_definition_fingerprints"]["orders"]
23369            .as_str()
23370            .expect("changed fingerprint")
23371            .to_string();
23372        assert_ne!(first, changed);
23373
23374        worker.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
23375        worker
23376            .register()
23377            .await
23378            .expect("register replacement handler");
23379        assert_eq!(
23380            server.request_bodies("/api/worker/register")[2]["workflow_definition_fingerprints"],
23381            json!({})
23382        );
23383    }
23384
23385    #[tokio::test]
23386    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
23387        let server = MockWorkerServer::start();
23388        let control_only = Client::builder(server.base_url())
23389            .control_token(Some("control-secret".to_string()))
23390            .build()
23391            .expect("control client");
23392
23393        let error = control_only
23394            .register_worker("worker", "queue", vec![], vec![], 1, 1)
23395            .await
23396            .expect_err("control token must not authorize a worker request");
23397        assert!(matches!(
23398            error,
23399            Error::MissingRoleCredentials { role: "worker", .. }
23400        ));
23401        assert_eq!(server.request_count("/api/worker/register"), 0);
23402
23403        let worker_only = Client::builder(server.base_url())
23404            .worker_token(Some("worker-secret".to_string()))
23405            .build()
23406            .expect("worker client");
23407        let error = worker_only
23408            .health()
23409            .await
23410            .expect_err("worker token must not authorize a control request");
23411        assert!(matches!(
23412            error,
23413            Error::MissingRoleCredentials {
23414                role: "control",
23415                ..
23416            }
23417        ));
23418        assert_eq!(server.request_count("/api/health"), 0);
23419    }
23420
23421    #[tokio::test]
23422    async fn shared_token_supports_worker_and_control_planes() {
23423        let server = MockWorkerServer::start();
23424        let client = Client::builder(server.base_url())
23425            .token(Some("shared-secret".to_string()))
23426            .build()
23427            .expect("client");
23428
23429        client.health().await.expect("control request");
23430        client
23431            .register_worker("worker", "queue", vec![], vec![], 1, 1)
23432            .await
23433            .expect("worker request");
23434
23435        assert_eq!(
23436            server.authorization_for("/api/health").as_deref(),
23437            Some("Bearer shared-secret")
23438        );
23439        assert_eq!(
23440            server.control_protocol_for("/api/health").as_deref(),
23441            Some(CONTROL_PLANE_VERSION)
23442        );
23443        assert_eq!(
23444            server.authorization_for("/api/worker/register").as_deref(),
23445            Some("Bearer shared-secret")
23446        );
23447        assert_eq!(
23448            server
23449                .worker_protocol_for("/api/worker/register")
23450                .as_deref(),
23451            Some(WORKER_PROTOCOL_VERSION)
23452        );
23453    }
23454
23455    #[tokio::test]
23456    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
23457        let server = MockWorkerServer::start();
23458        let client = Client::builder(server.base_url())
23459            .timeout(Duration::from_secs(2))
23460            .build()
23461            .expect("client");
23462
23463        client
23464            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
23465            .await
23466            .expect("register");
23467        client
23468            .heartbeat_worker("capture-worker", 1, 1)
23469            .await
23470            .expect("heartbeat");
23471        client
23472            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23473            .await
23474            .expect("workflow poll");
23475        client
23476            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23477            .await
23478            .expect("activity poll");
23479
23480        for path in [
23481            "/api/worker/register",
23482            "/api/worker/heartbeat",
23483            "/api/worker/workflow-tasks/poll",
23484            "/api/worker/activity-tasks/poll",
23485        ] {
23486            assert_eq!(
23487                server.worker_protocol_for(path).as_deref(),
23488                Some(WORKER_PROTOCOL_VERSION),
23489                "unexpected protocol for {path}"
23490            );
23491        }
23492
23493        assert_eq!(
23494            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
23495            1
23496        );
23497        assert_eq!(
23498            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
23499            1
23500        );
23501        assert!(
23502            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
23503                .as_str()
23504                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
23505        );
23506        assert!(
23507            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
23508                .as_str()
23509                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
23510        );
23511    }
23512
23513    #[tokio::test]
23514    async fn query_task_endpoints_send_the_query_feature_protocol() {
23515        let server = MockWorkerServer::start();
23516        let client = Client::builder(server.base_url())
23517            .timeout(Duration::from_secs(2))
23518            .build()
23519            .expect("client");
23520
23521        client
23522            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23523            .await
23524            .expect("query poll");
23525        client
23526            .complete_query_task(
23527                "query-capture",
23528                "capture-worker",
23529                1,
23530                json!(8),
23531                DEFAULT_CODEC,
23532            )
23533            .await
23534            .expect("query complete");
23535        client
23536            .fail_query_task(
23537                "query-capture",
23538                "capture-worker",
23539                1,
23540                "failed",
23541                "query_rejected",
23542                "QueryFailed",
23543            )
23544            .await
23545            .expect("query fail");
23546
23547        for path in [
23548            "/api/worker/query-tasks/poll",
23549            "/api/worker/query-tasks/query-capture/complete",
23550            "/api/worker/query-tasks/query-capture/fail",
23551        ] {
23552            assert_eq!(
23553                server.worker_protocol_for(path).as_deref(),
23554                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
23555                "unexpected protocol for {path}"
23556            );
23557        }
23558
23559        assert_eq!(
23560            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
23561            1
23562        );
23563        assert!(
23564            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
23565                .as_str()
23566                .is_some_and(|id| id.starts_with("rust-query-poll-"))
23567        );
23568    }
23569
23570    #[tokio::test]
23571    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
23572        let server = MockWorkerServer::transient_worker_failures();
23573        let client = Client::builder(server.base_url())
23574            .timeout(Duration::from_secs(2))
23575            .build()
23576            .expect("client");
23577
23578        client
23579            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23580            .await
23581            .expect("workflow poll retry");
23582        client
23583            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23584            .await
23585            .expect("activity poll retry");
23586        client
23587            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23588            .await
23589            .expect("query poll retry");
23590
23591        for path in [
23592            "/api/worker/workflow-tasks/poll",
23593            "/api/worker/activity-tasks/poll",
23594            "/api/worker/query-tasks/poll",
23595        ] {
23596            let bodies = server.request_bodies(path);
23597            assert_eq!(bodies.len(), 2, "{path} must be retried once");
23598            assert_eq!(
23599                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
23600                "{path} must preserve the request binding across retry"
23601            );
23602        }
23603    }
23604
23605    #[tokio::test]
23606    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
23607        let server = MockWorkerServer::consecutive_poll_failures(2);
23608        let client = Client::builder(server.base_url())
23609            .timeout(Duration::from_secs(2))
23610            .build()
23611            .expect("client");
23612        let mut worker = Worker::new(client, "capture")
23613            .worker_id("capture-worker")
23614            .poll_timeout(Duration::from_millis(10))
23615            .retry_policy(WorkerRetryPolicy {
23616                max_retries: 2,
23617                initial_backoff: Duration::from_millis(1),
23618                max_backoff: Duration::from_millis(1),
23619            });
23620        worker.register_workflow(
23621            "capture.workflow",
23622            |_ctx, _input| async move { Ok(Value::Null) },
23623        );
23624        worker.register_activity(
23625            "capture.activity",
23626            |_ctx, _input| async move { Ok(Value::Null) },
23627        );
23628        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23629            Ok(Value::Null)
23630        });
23631
23632        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23633
23634        for path in [
23635            "/api/worker/workflow-tasks/poll",
23636            "/api/worker/activity-tasks/poll",
23637            "/api/worker/query-tasks/poll",
23638        ] {
23639            let bodies = server.request_bodies(path);
23640            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23641            assert!(
23642                bodies
23643                    .iter()
23644                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23645                "{path} must preserve one request binding across every retry"
23646            );
23647        }
23648    }
23649
23650    fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23651        let reason = if unavailable {
23652            "storage_admission_unavailable"
23653        } else {
23654            "storage_pressure"
23655        };
23656        let mut body = json!({
23657            "reason": reason,
23658            "storage_state": if unavailable { "fenced" } else { "draining" },
23659            "retryable": true,
23660            "retry_after_seconds": 1,
23661        });
23662        if !mid_poll {
23663            body["request_admitted"] = json!(false);
23664        }
23665        if let Some(id) = poll_id {
23666            body["task"] = Value::Null;
23667            body["poll_status"] = json!(reason);
23668            body["poll_request_id"] = json!(id);
23669            body["retry_same_poll_request_id"] = json!(true);
23670            body["claim_admitted"] = json!(false);
23671        }
23672        body
23673    }
23674
23675    fn backend_refusal(path: &str, request: &str) -> Option<Value> {
23676        let operation = match path {
23677            "/api/worker/register" => "register_worker",
23678            "/api/worker/heartbeat" => "heartbeat_worker",
23679            "/api/worker/workflow-tasks/poll" => "poll_workflow_task",
23680            "/api/worker/activity-tasks/poll" => "poll_activity_task",
23681            "/api/worker/query-tasks/poll" => "poll_query_task",
23682            "/api/worker/update-validation-tasks/poll" => "poll_update_validation_task",
23683            _ => return None,
23684        };
23685        let request: Value = serde_json::from_str(request).ok()?;
23686        let mut response = json!({
23687            "reason": "backend_unavailable",
23688            "operation": operation,
23689            "outcome": "unknown",
23690            "retryable": true,
23691            "retry_after_seconds": 1,
23692            "worker_id": request["worker_id"],
23693            "task_queue": request.get("task_queue"),
23694        });
23695        if path.ends_with("/poll") {
23696            response["task"] = Value::Null;
23697            response["poll_status"] = json!("backend_unavailable");
23698            response["poll_request_id"] = request["poll_request_id"].clone();
23699            response["retry_same_poll_request_id"] = json!(true);
23700        }
23701        Some(response)
23702    }
23703
23704    fn backend_retry_override(
23705        path: &str,
23706        request: &str,
23707        number: usize,
23708    ) -> Option<(&'static str, String)> {
23709        if number > 8 {
23710            return None;
23711        }
23712        Some((
23713            "503 Service Unavailable",
23714            backend_refusal(path, request)?.to_string(),
23715        ))
23716    }
23717
23718    fn backend_unavailable_override(
23719        path: &str,
23720        request: &str,
23721        _number: usize,
23722    ) -> Option<(&'static str, String)> {
23723        Some((
23724            "503 Service Unavailable",
23725            backend_refusal(path, request)?.to_string(),
23726        ))
23727    }
23728
23729    fn storage_worker(server: &MockWorkerServer) -> Worker {
23730        Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23731            .worker_id("storage-worker")
23732            .retry_policy(WorkerRetryPolicy {
23733                max_retries: 1,
23734                initial_backoff: Duration::from_millis(1),
23735                max_backoff: Duration::from_millis(1),
23736            })
23737    }
23738
23739    fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23740        let requests = server.requests.lock().expect("requests");
23741        let bodies: Vec<_> = requests
23742            .iter()
23743            .filter(|request| request.path == path)
23744            .map(|request| &request.body)
23745            .collect();
23746        assert_eq!(bodies.len(), count, "{path}");
23747        assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23748    }
23749
23750    #[test]
23751    fn backend_recovery_requires_the_explicit_worker_contract() {
23752        for (path, operation) in [
23753            ("/worker/register", "register_worker"),
23754            ("/worker/heartbeat", "heartbeat_worker"),
23755            ("/worker/workflow-tasks/poll", "poll_workflow_task"),
23756            ("/worker/activity-tasks/poll", "poll_activity_task"),
23757            ("/worker/query-tasks/poll", "poll_query_task"),
23758            (
23759                "/worker/update-validation-tasks/poll",
23760                "poll_update_validation_task",
23761            ),
23762        ] {
23763            let poll_id = path.ends_with("/poll").then_some("same-poll");
23764            let request = json!({"worker_id":"same-worker","task_queue":"same-queue","poll_request_id":poll_id});
23765            let body = backend_refusal(&format!("/api{path}"), &request.to_string())
23766                .expect("worker operation");
23767            assert_eq!(body["operation"], operation);
23768            let error = Error::Http {
23769                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23770                body: body.to_string(),
23771            };
23772            assert_eq!(
23773                worker_backend_unavailable_retry_after(&error, path, poll_id),
23774                Some(Duration::from_secs(1))
23775            );
23776            assert!(!worker_operation_is_retryable(&error));
23777            for (field, value) in [
23778                ("operation", json!("wrong_operation")),
23779                ("outcome", json!("completed")),
23780                ("retryable", json!(false)),
23781                ("retry_after_seconds", json!(0)),
23782                ("worker_id", json!("")),
23783            ] {
23784                let mut invalid = body.clone();
23785                invalid[field] = value;
23786                let error = Error::Http {
23787                    status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23788                    body: invalid.to_string(),
23789                };
23790                assert!(
23791                    worker_backend_unavailable_retry_after(&error, path, poll_id).is_none(),
23792                    "{path}: {field}"
23793                );
23794                assert!(!worker_operation_is_retryable(&error));
23795            }
23796            if let Some(poll_id) = poll_id {
23797                for (field, value) in [
23798                    ("poll_request_id", json!("wrong-poll")),
23799                    ("poll_status", json!("empty")),
23800                    ("task", json!({"task_id":"claimed"})),
23801                    ("retry_same_poll_request_id", json!(false)),
23802                ] {
23803                    let mut invalid = body.clone();
23804                    invalid[field] = value;
23805                    let error = Error::Http {
23806                        status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23807                        body: invalid.to_string(),
23808                    };
23809                    assert!(
23810                        worker_backend_unavailable_retry_after(&error, path, Some(poll_id))
23811                            .is_none(),
23812                        "{path}: {field}"
23813                    );
23814                }
23815            }
23816            let error = Error::Http {
23817                status: reqwest::StatusCode::UNAUTHORIZED,
23818                body: body.to_string(),
23819            };
23820            assert!(worker_backend_unavailable_retry_after(&error, path, poll_id).is_none());
23821            assert!(!worker_operation_is_retryable(&error));
23822        }
23823    }
23824
23825    #[tokio::test]
23826    async fn backend_recovery_outlives_generic_retry_budget_and_preserves_polls() {
23827        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23828            request_override: Some(backend_retry_override),
23829            ..MockWorkerBehavior::default()
23830        });
23831        let mut worker =
23832            storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23833        worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23834        worker.register_activity("backend.activity", |_, _| async { Ok(Value::Null) });
23835        worker.register_query("backend.workflow", "state", |_, _| async {
23836            Ok(Value::Null)
23837        });
23838        worker.register().await.expect("registration recovery");
23839        worker
23840            .client
23841            .heartbeat_worker("storage-worker", 1, 1)
23842            .await
23843            .expect("heartbeat recovery");
23844        assert_eq!(worker.run_once().await.expect("poll recovery"), 0);
23845        for path in [
23846            "/api/worker/register",
23847            "/api/worker/heartbeat",
23848            "/api/worker/workflow-tasks/poll",
23849            "/api/worker/activity-tasks/poll",
23850            "/api/worker/query-tasks/poll",
23851        ] {
23852            assert_identical_requests(&server, path, 9);
23853        }
23854    }
23855
23856    #[tokio::test]
23857    async fn backend_recovery_wait_stops_with_worker() {
23858        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23859            request_override: Some(backend_unavailable_override),
23860            ..MockWorkerBehavior::default()
23861        });
23862        let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy {
23863            max_retries: 1,
23864            initial_backoff: Duration::from_secs(2),
23865            max_backoff: Duration::from_secs(2),
23866        });
23867        worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23868        let started = tokio::time::Instant::now();
23869        let _ = tokio::time::timeout(
23870            Duration::from_secs(1),
23871            worker.run_until(tokio::time::sleep(Duration::from_millis(100))),
23872        )
23873        .await
23874        .expect("shutdown interrupts backend wait");
23875        assert!(started.elapsed() < Duration::from_secs(1));
23876        assert!(server.request_count("/api/worker/register") >= 1);
23877    }
23878
23879    #[test]
23880    fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23881        for unavailable in [false, true] {
23882            for mid_poll in [false, true] {
23883                let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23884                let error = Error::Http {
23885                    status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23886                    body: body.to_string(),
23887                };
23888                assert_eq!(
23889                    worker_storage_admission_retry_after(&error, Some("same-poll")),
23890                    Some(Duration::from_secs(1))
23891                );
23892                assert!(
23893                    !worker_operation_is_retryable(&error),
23894                    "storage is not a bounded generic retry"
23895                );
23896                for (field, value) in [
23897                    ("poll_request_id", json!("wrong-poll")),
23898                    ("task", json!({"task_id":"claimed"})),
23899                    ("retryable", json!(false)),
23900                    ("retry_after_seconds", json!(0)),
23901                    ("retry_after_seconds", json!(true)),
23902                    ("retry_after_seconds", json!(1.0)),
23903                    ("storage_state", json!("normal")),
23904                    ("poll_status", json!("empty")),
23905                    ("claim_admitted", json!(true)),
23906                    ("retry_same_poll_request_id", json!(false)),
23907                    ("request_admitted", json!(true)),
23908                ] {
23909                    let mut invalid = body.clone();
23910                    invalid[field] = value;
23911                    let error = Error::Http {
23912                        status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23913                        body: invalid.to_string(),
23914                    };
23915                    assert!(
23916                        worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23917                        "{field}"
23918                    );
23919                }
23920            }
23921        }
23922        let body = storage_refusal(None, false, false);
23923        let error = Error::Http {
23924            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23925            body: body.to_string(),
23926        };
23927        assert!(worker_storage_admission_retry_after(&error, None).is_some());
23928        assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23929        let error = Error::Http {
23930            status: reqwest::StatusCode::FORBIDDEN,
23931            body: body.to_string(),
23932        };
23933        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23934        let body = storage_refusal(None, false, true);
23935        let error = Error::Http {
23936            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23937            body: body.to_string(),
23938        };
23939        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23940    }
23941
23942    #[tokio::test]
23943    async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23944        for unavailable in [false, true] {
23945            for mid_poll in [false, true] {
23946                let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23947                    poll_failures_per_path: 1,
23948                    storage_refusals: 7,
23949                    storage_path: Some("/poll"),
23950                    storage_unavailable: unavailable,
23951                    storage_mid_poll: mid_poll,
23952                    ..MockWorkerBehavior::default()
23953                });
23954                let mut worker = storage_worker(&server);
23955                worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23956                assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23957                for path in [
23958                    "/api/worker/workflow-tasks/poll",
23959                    "/api/worker/activity-tasks/poll",
23960                    "/api/worker/query-tasks/poll",
23961                ] {
23962                    assert_identical_requests(&server, path, 9);
23963                }
23964            }
23965        }
23966    }
23967
23968    #[tokio::test]
23969    async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23970        struct CountedBody(Arc<AtomicUsize>);
23971        impl Serialize for CountedBody {
23972            fn serialize<S: Serializer>(
23973                &self,
23974                serializer: S,
23975            ) -> std::result::Result<S::Ok, S::Error> {
23976                let count = self.0.fetch_add(1, Ordering::SeqCst);
23977                json!({"serialization":count,"lease_owner":"worker","attempt":7})
23978                    .serialize(serializer)
23979            }
23980        }
23981        for path in [
23982            "/api/worker/register",
23983            "/api/worker/heartbeat",
23984            "/api/worker/workflow-tasks/storage-task/complete",
23985            "/api/worker/workflow-tasks/storage-task/fail",
23986            "/api/worker/activity-tasks/storage-task/complete",
23987            "/api/worker/activity-tasks/storage-task/fail",
23988            "/api/worker/activity-tasks/storage-task/heartbeat",
23989            "/api/worker/query-tasks/storage-task/complete",
23990            "/api/worker/query-tasks/storage-task/fail",
23991        ] {
23992            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23993                storage_refusals: 7,
23994                storage_path: Some(path),
23995                ..MockWorkerBehavior::default()
23996            });
23997            let worker =
23998                storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23999            let calls = Arc::new(AtomicUsize::new(0));
24000            let _: Value = worker
24001                .client
24002                .request_json(
24003                    reqwest::Method::POST,
24004                    &path[4..],
24005                    RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
24006                    Some(&CountedBody(Arc::clone(&calls))),
24007                )
24008                .await
24009                .expect("prepared request recovery");
24010            assert_eq!(calls.load(Ordering::SeqCst), 1);
24011            assert_identical_requests(&server, path, 8);
24012        }
24013        for worker_scope in [false, true] {
24014            let path = if worker_scope {
24015                "/api/health"
24016            } else {
24017                "/api/worker/register"
24018            };
24019            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24020                storage_refusals: usize::MAX,
24021                storage_path: Some(path),
24022                ..MockWorkerBehavior::default()
24023            });
24024            let worker = storage_worker(&server);
24025            let client = worker.client.clone();
24026            let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
24027            let error = if worker_scope {
24028                worker
24029                    .client
24030                    .health()
24031                    .await
24032                    .expect_err("control plane is not retried")
24033            } else {
24034                client
24035                    .request_json::<Value, Value>(
24036                        reqwest::Method::POST,
24037                        "/worker/register",
24038                        RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
24039                        Some(&json!({})),
24040                    )
24041                    .await
24042                    .expect_err("direct client is not retried")
24043            };
24044            assert!(worker_storage_admission_body(&error).is_some());
24045            assert_eq!(server.request_count(path), 1);
24046        }
24047    }
24048
24049    #[tokio::test]
24050    async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
24051        for fail in [false, true] {
24052            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24053                storage_activity: true,
24054                storage_refusals: 7,
24055                storage_path: Some("/storage-activity/"),
24056                ..MockWorkerBehavior::default()
24057            });
24058            let mut worker = storage_worker(&server);
24059            let calls = Arc::new(AtomicUsize::new(0));
24060            let observed = Arc::clone(&calls);
24061            worker.register_activity("storage.activity", move |ctx, _| {
24062                let calls = Arc::clone(&observed);
24063                async move {
24064                    calls.fetch_add(1, Ordering::SeqCst);
24065                    ctx.heartbeat(json!({"step":2})).await?;
24066                    if fail {
24067                        Err(Error::WorkerLoop("intentional handler failure".to_string()))
24068                    } else {
24069                        Ok(json!({"receipt":true}))
24070                    }
24071                }
24072            });
24073            assert_eq!(worker.run_once().await.expect("activity settled"), 1);
24074            assert_eq!(calls.load(Ordering::SeqCst), 1);
24075            assert_identical_requests(
24076                &server,
24077                "/api/worker/activity-tasks/storage-activity/heartbeat",
24078                8,
24079            );
24080            let suffix = if fail { "fail" } else { "complete" };
24081            assert_identical_requests(
24082                &server,
24083                &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
24084                8,
24085            );
24086            let other = if fail { "complete" } else { "fail" };
24087            assert_eq!(
24088                server.request_count(&format!(
24089                    "/api/worker/activity-tasks/storage-activity/{other}"
24090                )),
24091                0
24092            );
24093        }
24094    }
24095
24096    #[tokio::test]
24097    async fn storage_waits_are_interruptible_without_false_activity_failure() {
24098        for path in [
24099            "/api/worker/register",
24100            "/api/worker/heartbeat",
24101            "/api/worker/activity-tasks/poll",
24102            "/api/worker/activity-tasks/storage-activity/heartbeat",
24103            "/api/worker/activity-tasks/storage-activity/complete",
24104        ] {
24105            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24106                storage_activity: true,
24107                storage_refusals: usize::MAX,
24108                storage_path: Some(path),
24109                ..MockWorkerBehavior::default()
24110            });
24111            let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
24112            worker.register_activity("storage.activity", |ctx, _| async move {
24113                ctx.heartbeat(json!({"step":2})).await?;
24114                Ok(json!({"receipt":true}))
24115            });
24116            let shutdown = async {
24117                while server.request_count(path) == 0 {
24118                    tokio::time::sleep(Duration::from_millis(1)).await;
24119                }
24120            };
24121            let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
24122                .await
24123                .expect("shutdown interrupts admission");
24124            assert!(
24125                result.is_err(),
24126                "a refused operation must not appear acknowledged: {path}, {result:?}"
24127            );
24128            assert_eq!(server.request_count(path), 1);
24129            assert_eq!(
24130                server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
24131                0
24132            );
24133            assert_eq!(
24134                server.request_count("/api/worker/registrations/mock-worker"),
24135                usize::from(!path.ends_with("/register"))
24136            );
24137        }
24138    }
24139
24140    #[tokio::test]
24141    async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
24142        for fail in [false, true] {
24143            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24144                storage_query: true,
24145                storage_refusals: 7,
24146                storage_path: Some("/storage-query/"),
24147                ..MockWorkerBehavior::default()
24148            });
24149            let mut worker = storage_worker(&server);
24150            worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
24151            let calls = Arc::new(AtomicUsize::new(0));
24152            let observed = Arc::clone(&calls);
24153            worker.register_query("storage.workflow", "state", move |_, _| {
24154                let calls = Arc::clone(&observed);
24155                async move {
24156                    calls.fetch_add(1, Ordering::SeqCst);
24157                    if fail {
24158                        Err(Error::WorkerLoop("intentional query failure".to_string()))
24159                    } else {
24160                        Ok(json!({"state":"waiting"}))
24161                    }
24162                }
24163            });
24164            assert_eq!(worker.run_once().await.expect("query settled"), 1);
24165            assert_eq!(calls.load(Ordering::SeqCst), 1);
24166            let suffix = if fail { "fail" } else { "complete" };
24167            assert_identical_requests(
24168                &server,
24169                &format!("/api/worker/query-tasks/storage-query/{suffix}"),
24170                8,
24171            );
24172            let other = if fail { "complete" } else { "fail" };
24173            assert_eq!(
24174                server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
24175                0
24176            );
24177        }
24178    }
24179
24180    #[tokio::test]
24181    async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
24182        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24183            storage_refusals: 7,
24184            storage_path: Some("/poll"),
24185            unauthorized_polls: true,
24186            ..MockWorkerBehavior::default()
24187        });
24188        let error = storage_worker(&server)
24189            .run_once()
24190            .await
24191            .expect_err("auth remains terminal");
24192        assert!(matches!(
24193            error,
24194            Error::Http {
24195                status: reqwest::StatusCode::UNAUTHORIZED,
24196                ..
24197            }
24198        ));
24199        assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
24200
24201        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24202            storage_refusals: 7,
24203            storage_path: Some("/activity-cancel/complete"),
24204            ..MockWorkerBehavior::default()
24205        });
24206        let worker =
24207            storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
24208        let error = worker
24209            .client
24210            .complete_activity_task(
24211                "activity-cancel",
24212                "attempt-cancel",
24213                "worker",
24214                json!({}),
24215                DEFAULT_CODEC,
24216            )
24217            .await
24218            .expect_err("cancellation remains terminal");
24219        assert!(activity_task_rejection_is_final(&error));
24220        assert_identical_requests(
24221            &server,
24222            "/api/worker/activity-tasks/activity-cancel/complete",
24223            8,
24224        );
24225
24226        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24227            storage_refusals: usize::MAX,
24228            storage_path: Some("/poll"),
24229            storage_wrong_poll_id: true,
24230            ..MockWorkerBehavior::default()
24231        });
24232        assert!(storage_worker(&server).run_once().await.is_err());
24233        assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
24234    }
24235
24236    #[tokio::test]
24237    async fn storage_pollers_stop_when_the_run_future_is_aborted() {
24238        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24239            storage_refusals: usize::MAX,
24240            storage_path: Some("/poll"),
24241            ..MockWorkerBehavior::default()
24242        });
24243        let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
24244        worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
24245        let run = tokio::spawn(async move { worker.run().await });
24246        tokio::time::timeout(Duration::from_secs(2), async {
24247            while server.request_count("/api/worker/activity-tasks/poll") == 0 {
24248                tokio::time::sleep(Duration::from_millis(1)).await;
24249            }
24250        })
24251        .await
24252        .expect("poll started");
24253        run.abort();
24254        assert!(run.await.expect_err("cancelled run").is_cancelled());
24255        tokio::time::sleep(Duration::from_millis(250)).await;
24256        assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
24257    }
24258
24259    #[tokio::test]
24260    async fn query_protocol_rejection_from_older_server_is_typed() {
24261        let server = MockWorkerServer::reject_query_protocol();
24262        let client = Client::builder(server.base_url())
24263            .timeout(Duration::from_secs(2))
24264            .build()
24265            .expect("client");
24266
24267        let error = client
24268            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
24269            .await
24270            .expect_err("server below query protocol floor must reject");
24271        let Error::Protocol(failure) = error else {
24272            panic!("expected typed protocol failure");
24273        };
24274
24275        assert_eq!(failure.status, 400);
24276        assert_eq!(failure.reason, "unsupported_protocol_version");
24277        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
24278        assert_eq!(
24279            failure.requested_version.as_deref(),
24280            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
24281        );
24282        assert_eq!(
24283            server
24284                .worker_protocol_for("/api/worker/query-tasks/poll")
24285                .as_deref(),
24286            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
24287        );
24288    }
24289
24290    #[tokio::test]
24291    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
24292        let server = MockWorkerServer::reject_query_protocol();
24293        let client = Client::builder(server.base_url())
24294            .timeout(Duration::from_secs(2))
24295            .build()
24296            .expect("client");
24297        let mut worker = Worker::new(client, "rust-workers")
24298            .worker_id("baseline-worker")
24299            .poll_timeout(Duration::from_millis(10));
24300
24301        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
24302            Ok(Value::Null)
24303        });
24304
24305        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
24306        assert_eq!(
24307            server
24308                .worker_protocol_for("/api/worker/workflow-tasks/poll")
24309                .as_deref(),
24310            Some(WORKER_PROTOCOL_VERSION)
24311        );
24312        assert_eq!(
24313            server.worker_protocol_for("/api/worker/query-tasks/poll"),
24314            None,
24315            "a worker without query handlers must not use the query-task endpoint"
24316        );
24317    }
24318
24319    #[tokio::test]
24320    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
24321        let server = MockWorkerServer::reject_query_completion();
24322        let client = Client::builder(server.base_url())
24323            .timeout(Duration::from_secs(2))
24324            .build()
24325            .expect("client");
24326
24327        let error = client
24328            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
24329            .await
24330            .expect_err("expired completion must be rejected");
24331        let Error::QueryFailed(failure) = error else {
24332            panic!("expected typed query failure");
24333        };
24334        assert_eq!(failure.status, 409);
24335        assert_eq!(failure.reason, "query_task_timed_out");
24336
24337        let mut worker = Worker::new(client, "rust-workers")
24338            .worker_id("late-worker")
24339            .poll_timeout(Duration::from_millis(10));
24340        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24341        worker.register_query(
24342            "counter",
24343            "current",
24344            |_ctx, _args| async move { Ok(json!(8)) },
24345        );
24346
24347        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
24348        assert_eq!(
24349            worker
24350                .run_once()
24351                .await
24352                .expect("worker continues after late completion"),
24353            0
24354        );
24355        assert_eq!(
24356            server.request_count("/api/worker/query-tasks/query-late/complete"),
24357            2
24358        );
24359        assert_eq!(
24360            server.request_count("/api/worker/query-tasks/query-late/fail"),
24361            0,
24362            "a server completion rejection must not be reported as an encoding failure"
24363        );
24364    }
24365
24366    #[tokio::test]
24367    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
24368        let server = MockWorkerServer::start();
24369        let client = Client::builder(server.base_url())
24370            .timeout(Duration::from_secs(2))
24371            .build()
24372            .expect("client");
24373        let mut worker = Worker::new(client, "rust-workers")
24374            .worker_id("joined-worker")
24375            .poll_timeout(Duration::from_millis(10));
24376        worker.register_workflow(
24377            "joined.workflow",
24378            |_ctx, _input| async move { Ok(Value::Null) },
24379        );
24380        worker.register_activity(
24381            "joined.activity",
24382            |_ctx, _input| async move { Ok(Value::Null) },
24383        );
24384        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
24385            Ok(Value::Null)
24386        });
24387
24388        worker
24389            .run_until(tokio::time::sleep(Duration::from_millis(20)))
24390            .await
24391            .expect("normal shutdown");
24392
24393        let deregistration_path = "/api/worker/registrations/mock-worker";
24394        assert_eq!(server.request_count(deregistration_path), 1);
24395        for poll_path in [
24396            "/api/worker/workflow-tasks/poll",
24397            "/api/worker/activity-tasks/poll",
24398            "/api/worker/query-tasks/poll",
24399        ] {
24400            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
24401        }
24402        assert_eq!(
24403            server.captured_paths().last().map(String::as_str),
24404            Some(deregistration_path),
24405            "deregistration must start only after every poller has joined"
24406        );
24407    }
24408
24409    #[tokio::test]
24410    async fn registration_failure_does_not_deregister() {
24411        let server = MockWorkerServer::rejected_registration();
24412        let client = Client::builder(server.base_url())
24413            .timeout(Duration::from_secs(2))
24414            .build()
24415            .expect("client");
24416        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
24417
24418        let error = worker
24419            .run_until(async {})
24420            .await
24421            .expect_err("registration must fail");
24422        assert!(matches!(
24423            error,
24424            Error::Http {
24425                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
24426                ..
24427            }
24428        ));
24429        assert!(server
24430            .captured_paths()
24431            .iter()
24432            .all(|path| !path.starts_with("/api/worker/registrations/")));
24433    }
24434
24435    #[tokio::test]
24436    async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
24437        let server = MockWorkerServer::rejected_registration_protocol();
24438        let client = Client::builder(server.base_url())
24439            .timeout(Duration::from_secs(2))
24440            .build()
24441            .expect("client");
24442        let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
24443
24444        let error = worker
24445            .run_until(async {})
24446            .await
24447            .expect_err("a protocol 1.16 server must reject this worker");
24448        let Error::Protocol(failure) = error else {
24449            panic!("expected typed protocol rejection");
24450        };
24451        assert_eq!(failure.reason, "unsupported_protocol_version");
24452        assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
24453        assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
24454        assert_eq!(
24455            server
24456                .worker_protocol_for("/api/worker/register")
24457                .as_deref(),
24458            Some(WORKER_PROTOCOL_VERSION)
24459        );
24460    }
24461
24462    #[tokio::test]
24463    async fn declined_registration_does_not_deregister() {
24464        let server = MockWorkerServer::declined_registration();
24465        let client = Client::builder(server.base_url())
24466            .timeout(Duration::from_secs(2))
24467            .build()
24468            .expect("client");
24469        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
24470
24471        let error = worker
24472            .run_until(async {})
24473            .await
24474            .expect_err("declined registration must fail");
24475        assert!(matches!(error, Error::WorkerLoop(_)));
24476        assert!(error.to_string().contains("was not accepted"));
24477        assert!(server
24478            .captured_paths()
24479            .iter()
24480            .all(|path| !path.starts_with("/api/worker/registrations/")));
24481    }
24482
24483    #[tokio::test]
24484    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
24485        let server = MockWorkerServer::rejected_deregistration();
24486        let client = Client::builder(server.base_url())
24487            .timeout(Duration::from_secs(2))
24488            .build()
24489            .expect("client");
24490        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
24491
24492        let error = worker
24493            .run_until(async {})
24494            .await
24495            .expect_err("deregistration must fail");
24496        assert!(matches!(
24497            error,
24498            Error::Http {
24499                status: reqwest::StatusCode::FORBIDDEN,
24500                ..
24501            }
24502        ));
24503        assert_eq!(
24504            server.request_count("/api/worker/registrations/mock-worker"),
24505            1
24506        );
24507    }
24508
24509    #[tokio::test]
24510    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
24511        let server = MockWorkerServer::rejected_deregistration_protocol();
24512        let client = Client::builder(server.base_url())
24513            .timeout(Duration::from_secs(2))
24514            .build()
24515            .expect("client");
24516        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
24517
24518        let error = worker
24519            .run_until(async {})
24520            .await
24521            .expect_err("protocol rejection must fail shutdown");
24522        let Error::Protocol(failure) = error else {
24523            panic!("expected typed protocol failure");
24524        };
24525        assert_eq!(failure.reason, "unsupported_protocol_version");
24526        assert_eq!(
24527            failure.requested_version.as_deref(),
24528            Some(WORKER_PROTOCOL_VERSION)
24529        );
24530        assert_eq!(
24531            server.request_count("/api/worker/registrations/mock-worker"),
24532            1
24533        );
24534    }
24535
24536    #[tokio::test]
24537    async fn primary_poller_error_retains_deregistration_failure_context() {
24538        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
24539        let client = Client::builder(server.base_url())
24540            .timeout(Duration::from_secs(2))
24541            .build()
24542            .expect("client");
24543        let mut worker = Worker::new(client, "rust-workers")
24544            .worker_id("combined-failure")
24545            .poll_timeout(Duration::from_millis(10));
24546        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
24547            Ok(Value::Null)
24548        });
24549
24550        let error = worker
24551            .run()
24552            .await
24553            .expect_err("worker and cleanup must fail");
24554        let summary = error.to_string();
24555        assert!(summary.contains("authentication_failed"));
24556        assert!(summary.contains("worker cannot deregister"));
24557        let Error::WorkerShutdown {
24558            primary,
24559            deregistration,
24560        } = error
24561        else {
24562            panic!("expected combined worker shutdown error");
24563        };
24564        assert!(matches!(
24565            *primary,
24566            Error::Http {
24567                status: reqwest::StatusCode::UNAUTHORIZED,
24568                ..
24569            }
24570        ));
24571        assert!(matches!(
24572            *deregistration,
24573            Error::Http {
24574                status: reqwest::StatusCode::FORBIDDEN,
24575                ..
24576            }
24577        ));
24578        assert_eq!(
24579            server.request_count("/api/worker/registrations/mock-worker"),
24580            1
24581        );
24582    }
24583
24584    #[tokio::test]
24585    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
24586        let server = MockWorkerServer::start();
24587        let client = Client::builder(server.base_url())
24588            .timeout(Duration::from_secs(2))
24589            .build()
24590            .expect("client");
24591        let mut worker = Worker::new(client, "rust-workers")
24592            .worker_id("activity-only-worker")
24593            .poll_timeout(Duration::from_millis(10));
24594
24595        worker.register_activity(
24596            "activity.only",
24597            |_ctx, _args| async move { Ok(Value::Null) },
24598        );
24599
24600        worker.run_until(async {}).await.expect("run worker");
24601    }
24602
24603    #[tokio::test]
24604    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
24605        let server = MockWorkerServer::start();
24606        let client = Client::builder(server.base_url())
24607            .timeout(Duration::from_secs(2))
24608            .build()
24609            .expect("client");
24610        let mut worker = Worker::new(client, "rust-workers")
24611            .worker_id("workflow-only-worker")
24612            .poll_timeout(Duration::from_millis(10));
24613
24614        worker.register_workflow(
24615            "workflow.only",
24616            |_ctx, _input| async move { Ok(Value::Null) },
24617        );
24618
24619        worker.run_until(async {}).await.expect("run worker");
24620    }
24621
24622    #[tokio::test]
24623    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
24624        let server = MockWorkerServer::start();
24625        let client = Client::builder(server.base_url())
24626            .timeout(Duration::from_secs(2))
24627            .build()
24628            .expect("client");
24629        let observations = Arc::new(Mutex::new(Vec::new()));
24630        let observed = Arc::clone(&observations);
24631        let mut worker = Worker::new(client, "rust-workers")
24632            .worker_id("observed-heartbeat-worker")
24633            .poll_timeout(Duration::from_millis(10))
24634            .on_worker_heartbeat(move |observation| {
24635                observed
24636                    .lock()
24637                    .expect("heartbeat observations")
24638                    .push(observation.clone());
24639            });
24640
24641        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
24642            Ok(Value::Null)
24643        });
24644        let acknowledged = Arc::clone(&observations);
24645        worker
24646            .run_until(async move {
24647                tokio::time::timeout(Duration::from_secs(2), async move {
24648                    loop {
24649                        if !acknowledged
24650                            .lock()
24651                            .expect("heartbeat observations")
24652                            .is_empty()
24653                        {
24654                            break;
24655                        }
24656                        tokio::time::sleep(Duration::from_millis(1)).await;
24657                    }
24658                })
24659                .await
24660                .expect("heartbeat acknowledgement within timeout");
24661            })
24662            .await
24663            .expect("run worker");
24664
24665        let observations = observations.lock().expect("heartbeat observations");
24666        let first = observations.first().expect("heartbeat acknowledgement");
24667        assert_eq!(first.worker_id, "observed-heartbeat-worker");
24668        assert_eq!(first.task_queue, "rust-workers");
24669        assert!(first.acknowledged_at_unix_millis > 0);
24670        assert_eq!(first.acknowledgement, json!({}));
24671    }
24672
24673    #[tokio::test]
24674    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
24675        let server = MockWorkerServer::delayed_heartbeat_worker();
24676        let client = Client::builder(server.base_url())
24677            .timeout(Duration::from_secs(3))
24678            .build()
24679            .expect("client");
24680        let observations = Arc::new(Mutex::new(Vec::new()));
24681        let observed = Arc::clone(&observations);
24682        let mut worker = Worker::new(client, "rust-snapshot-workers")
24683            .worker_id("rust-snapshot-worker")
24684            .poll_timeout(Duration::from_millis(10))
24685            .on_worker_heartbeat(move |observation| {
24686                observed
24687                    .lock()
24688                    .expect("heartbeat observations")
24689                    .push(observation.clone());
24690            });
24691
24692        worker.register_workflow("snapshot", |ctx, _input| async move {
24693            ctx.wait_signal("finish").await?;
24694            Ok(json!({"status": "finished"}))
24695        });
24696        worker.register_query("snapshot", "current", |ctx, _args| async move {
24697            Ok(json!(ctx
24698                .signals("increment")
24699                .iter()
24700                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24701                .sum::<i64>()))
24702        });
24703        worker.register_activity("cancel-aware", |_ctx, _args| async move {
24704            Ok(json!({"late": "completion"}))
24705        });
24706
24707        worker
24708            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
24709            .await
24710            .expect("delayed heartbeat must allow a clean worker shutdown");
24711
24712        let observations = observations.lock().expect("heartbeat observations");
24713        assert!(
24714            observations.len() >= 3,
24715            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
24716        );
24717        assert!(
24718            observations.windows(2).all(|pair| {
24719                pair[1].acknowledged_at_unix_millis
24720                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
24721                    >= 850
24722            }),
24723            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
24724        );
24725        drop(observations);
24726
24727        let heartbeat_times = server.request_times("/api/worker/heartbeat");
24728        let delayed_request_at = *heartbeat_times
24729            .get(1)
24730            .expect("intentionally delayed heartbeat request");
24731        let delay_window_start = delayed_request_at + Duration::from_millis(100);
24732        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
24733        for path in [
24734            "/api/worker/workflow-tasks/poll",
24735            "/api/worker/activity-tasks/poll",
24736            "/api/worker/query-tasks/poll",
24737        ] {
24738            assert!(
24739                server
24740                    .request_times(path)
24741                    .iter()
24742                    .any(|received_at| *received_at >= delay_window_start
24743                        && *received_at <= delay_window_end),
24744                "{path} must keep polling while a heartbeat acknowledgement is delayed"
24745            );
24746        }
24747        assert!(
24748            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
24749            "workflow work must be settled"
24750        );
24751        assert!(
24752            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
24753            "activity work must be settled"
24754        );
24755        assert!(
24756            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
24757            "query work must be settled"
24758        );
24759    }
24760
24761    #[tokio::test]
24762    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
24763        let server = MockWorkerServer::heartbeat_retry_worker();
24764        let client = Client::builder(server.base_url())
24765            .timeout(Duration::from_secs(2))
24766            .build()
24767            .expect("client");
24768        let observations = Arc::new(Mutex::new(Vec::new()));
24769        let observed = Arc::clone(&observations);
24770        let worker = Worker::new(client, "rust-workers")
24771            .worker_id("heartbeat-retry-worker")
24772            .retry_policy(WorkerRetryPolicy {
24773                max_retries: 1,
24774                initial_backoff: Duration::from_millis(300),
24775                max_backoff: Duration::from_millis(300),
24776            })
24777            .on_worker_heartbeat(move |observation| {
24778                observed
24779                    .lock()
24780                    .expect("heartbeat observations")
24781                    .push(observation.clone());
24782            });
24783
24784        worker
24785            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
24786            .await
24787            .expect("retryable heartbeat failure must remain bounded and recover");
24788
24789        let observations = observations.lock().expect("heartbeat observations");
24790        assert!(observations.len() >= 3, "heartbeat retry must recover");
24791        assert!(
24792            observations.windows(2).all(|pair| {
24793                pair[1]
24794                    .acknowledged_at_unix_millis
24795                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
24796                    >= 850
24797            }),
24798            "a successful retry must start a fresh advertised cadence: {observations:?}"
24799        );
24800        assert_eq!(
24801            server.request_count("/api/worker/heartbeat"),
24802            observations.len() + 1,
24803            "one retryable failure must add exactly one bounded request"
24804        );
24805    }
24806
24807    #[tokio::test]
24808    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
24809        let server = MockWorkerServer::waiting_query_worker();
24810        let client = Client::builder(server.base_url())
24811            .timeout(Duration::from_secs(2))
24812            .build()
24813            .expect("client");
24814        let observations = Arc::new(Mutex::new(Vec::new()));
24815        let observed = Arc::clone(&observations);
24816        let mut worker = Worker::new(client, "rust-snapshot-workers")
24817            .worker_id("rust-snapshot-worker")
24818            .poll_timeout(Duration::from_millis(10))
24819            .on_worker_heartbeat(move |observation| {
24820                observed
24821                    .lock()
24822                    .expect("heartbeat observations")
24823                    .push(observation.clone());
24824            });
24825
24826        worker.register_workflow("snapshot", |ctx, _input| async move {
24827            ctx.wait_signal("finish").await?;
24828            Ok(json!({"status": "finished"}))
24829        });
24830        worker.register_query("snapshot", "current", |ctx, _args| async move {
24831            let current = ctx
24832                .signals("increment")
24833                .iter()
24834                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24835                .sum::<i64>();
24836            Ok(json!(current))
24837        });
24838        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24839
24840        worker
24841            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24842            .await
24843            .expect("pending workflow and query poller must remain live until shutdown");
24844
24845        assert!(
24846            observations.lock().expect("heartbeat observations").len() >= 4,
24847            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24848        );
24849        assert!(
24850            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24851            "workflow polling must continue after empty replay acknowledgements"
24852        );
24853        assert!(
24854            server.request_count("/api/worker/query-tasks/poll") >= 2,
24855            "query polling must continue after serving the current query"
24856        );
24857        assert_eq!(
24858            server.request_body("/api/worker/register")["capabilities"],
24859            json!([
24860                CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24861                DURABLE_SELECTION_CAPABILITY,
24862                MEMO_UPSERTS_CAPABILITY,
24863                TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24864                QUERY_TASKS_CAPABILITY,
24865                WORKFLOW_UPDATES_CAPABILITY,
24866                MESSAGE_STREAMS_CAPABILITY
24867            ])
24868        );
24869        assert_eq!(
24870            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24871            json!({
24872                "queries": ["current"],
24873                "query_contracts": [],
24874                "signals": [],
24875                "signal_contracts": [],
24876                "updates": ["replace"],
24877                "update_contracts": [],
24878                "update_validators": [],
24879            })
24880        );
24881
24882        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24883        assert_eq!(
24884            opened["commands"],
24885            json!([{
24886                "type": "open_signal_wait",
24887                "signal_name": "finish",
24888            }])
24889        );
24890
24891        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24892            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24893            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24894            let failure = server.request_body(&fail_path);
24895            assert_eq!(
24896                failure["failure"]["type"],
24897                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24898            );
24899            assert_eq!(server.request_count(&completion_path), 0);
24900        }
24901
24902        let query_completion =
24903            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24904        assert_eq!(query_completion["result"], json!(8));
24905
24906        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24907        assert_eq!(
24908            server.request_count(terminal_path),
24909            1,
24910            "the matching signal must settle the workflow exactly once"
24911        );
24912        let terminal = server.request_body(terminal_path);
24913        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24914        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24915        assert_eq!(
24916            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24917                .expect("terminal workflow result"),
24918            json!({"status": "finished"})
24919        );
24920    }
24921
24922    #[tokio::test]
24923    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24924        let server = MockWorkerServer::transient_worker_failures();
24925        let client = Client::builder(server.base_url())
24926            .timeout(Duration::from_secs(2))
24927            .build()
24928            .expect("client");
24929        let mut worker = Worker::new(client, "rust-workers")
24930            .worker_id("retry-worker")
24931            .poll_timeout(Duration::from_millis(10))
24932            .retry_policy(WorkerRetryPolicy {
24933                max_retries: 2,
24934                initial_backoff: Duration::from_millis(1),
24935                max_backoff: Duration::from_millis(1),
24936            });
24937        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24938        worker.register_activity(
24939            "counter.activity",
24940            |_ctx, _input| async move { Ok(Value::Null) },
24941        );
24942        worker.register_query(
24943            "counter",
24944            "current",
24945            |_ctx, _args| async move { Ok(json!(8)) },
24946        );
24947
24948        worker
24949            .run_until(tokio::time::sleep(Duration::from_millis(75)))
24950            .await
24951            .expect("transient failures must not stop the worker");
24952
24953        for path in [
24954            "/api/worker/heartbeat",
24955            "/api/worker/workflow-tasks/poll",
24956            "/api/worker/activity-tasks/poll",
24957            "/api/worker/query-tasks/poll",
24958        ] {
24959            assert!(
24960                server.request_count(path) >= 2,
24961                "{path} must continue after its transient failure"
24962            );
24963        }
24964    }
24965
24966    #[tokio::test]
24967    async fn worker_continues_after_long_poll_capacity_backpressure() {
24968        let server = MockWorkerServer::capacity_limited_activity_poll();
24969        let client = Client::builder(server.base_url())
24970            .timeout(Duration::from_secs(2))
24971            .build()
24972            .expect("client");
24973        let mut worker = Worker::new(client, "rust-workers")
24974            .worker_id("capacity-worker")
24975            .poll_timeout(Duration::from_millis(10))
24976            .retry_policy(WorkerRetryPolicy {
24977                max_retries: 0,
24978                initial_backoff: Duration::from_millis(1),
24979                max_backoff: Duration::from_millis(1),
24980            });
24981        worker.register_activity("capacity.activity", |_ctx, _input| async move {
24982            Ok(json!({"handled": true}))
24983        });
24984
24985        worker
24986            .run_until(tokio::time::sleep(Duration::from_millis(50)))
24987            .await
24988            .expect("capacity backpressure must not stop the worker");
24989
24990        assert!(
24991            server.request_count("/api/worker/activity-tasks/poll") >= 2,
24992            "the activity poller must continue after capacity backpressure"
24993        );
24994        assert_eq!(
24995            server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24996            1,
24997            "the worker must complete work returned after capacity recovers"
24998        );
24999    }
25000
25001    #[test]
25002    fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
25003        let capacity = Error::Http {
25004            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
25005            body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
25006        };
25007        assert_eq!(
25008            worker_poll_capacity_retry_after(&capacity),
25009            Some(Duration::from_secs(3))
25010        );
25011
25012        let rejected_capacity = Error::Http {
25013            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
25014            body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
25015        };
25016        assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
25017        assert!(!worker_operation_is_retryable(&rejected_capacity));
25018
25019        let ordinary_rate_limit = Error::Http {
25020            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
25021            body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
25022                .to_string(),
25023        };
25024        assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
25025        assert!(worker_operation_is_retryable(&ordinary_rate_limit));
25026    }
25027
25028    #[tokio::test]
25029    async fn worker_bounds_transport_retries() {
25030        let server = MockWorkerServer::unavailable_polls();
25031        let client = Client::builder(server.base_url())
25032            .timeout(Duration::from_secs(2))
25033            .build()
25034            .expect("client");
25035        let mut worker = Worker::new(client, "rust-workers")
25036            .worker_id("bounded-retry-worker")
25037            .poll_timeout(Duration::from_millis(10))
25038            .retry_policy(WorkerRetryPolicy {
25039                max_retries: 2,
25040                initial_backoff: Duration::from_millis(1),
25041                max_backoff: Duration::from_millis(1),
25042            });
25043        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25044
25045        let error = worker.run().await.expect_err("retry bound must terminate");
25046        assert!(matches!(error, Error::Transport(_)));
25047        assert_eq!(
25048            server.request_count("/api/worker/workflow-tasks/poll"),
25049            3,
25050            "one initial request plus exactly two retries"
25051        );
25052    }
25053
25054    #[tokio::test]
25055    async fn worker_retry_policy_can_disable_poll_retries() {
25056        let server = MockWorkerServer::unavailable_polls();
25057        let client = Client::builder(server.base_url())
25058            .timeout(Duration::from_secs(2))
25059            .build()
25060            .expect("client");
25061        let mut worker = Worker::new(client, "rust-workers")
25062            .worker_id("no-retry-worker")
25063            .poll_timeout(Duration::from_millis(10))
25064            .retry_policy(WorkerRetryPolicy {
25065                max_retries: 0,
25066                initial_backoff: Duration::from_millis(1),
25067                max_backoff: Duration::from_millis(1),
25068            });
25069        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25070
25071        let error = worker
25072            .run_once()
25073            .await
25074            .expect_err("disabled retries must return the first transport failure");
25075        assert!(matches!(error, Error::Transport(_)));
25076        assert_eq!(
25077            server.request_count("/api/worker/workflow-tasks/poll"),
25078            1,
25079            "max_retries=0 must send only the initial request"
25080        );
25081    }
25082
25083    #[tokio::test]
25084    async fn worker_does_not_retry_authentication_failures() {
25085        let server = MockWorkerServer::unauthorized_polls();
25086        let client = Client::builder(server.base_url())
25087            .timeout(Duration::from_secs(2))
25088            .build()
25089            .expect("client");
25090        let mut worker = Worker::new(client, "rust-workers")
25091            .worker_id("unauthorized-worker")
25092            .poll_timeout(Duration::from_millis(10));
25093        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25094
25095        let error = worker
25096            .run()
25097            .await
25098            .expect_err("authentication must terminate");
25099        let Error::Http { status, body } = error else {
25100            panic!("expected stable HTTP authentication error");
25101        };
25102        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
25103        assert!(body.contains("authentication_failed"));
25104        assert_eq!(
25105            server.request_count("/api/worker/workflow-tasks/poll"),
25106            1,
25107            "authentication failures must not be retried"
25108        );
25109    }
25110
25111    #[derive(Clone, Debug)]
25112    struct CapturedRequest {
25113        headers: String,
25114        method: String,
25115        path: String,
25116        authorization: Option<String>,
25117        namespace: Option<String>,
25118        worker_protocol: Option<String>,
25119        control_protocol: Option<String>,
25120        body: String,
25121        received_at: Instant,
25122    }
25123
25124    struct MockWorkerServer {
25125        addr: SocketAddr,
25126        stop: Arc<AtomicBool>,
25127        requests: Arc<Mutex<Vec<CapturedRequest>>>,
25128        thread: Option<thread::JoinHandle<()>>,
25129    }
25130
25131    type RequestOverride = fn(&str, &str, usize) -> Option<(&'static str, String)>;
25132
25133    #[derive(Clone, Copy, Default)]
25134    struct MockWorkerBehavior {
25135        response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
25136        request_override: Option<RequestOverride>,
25137        storage_refusals: usize,
25138        storage_path: Option<&'static str>,
25139        storage_unavailable: bool,
25140        storage_mid_poll: bool,
25141        storage_activity: bool,
25142        storage_query: bool,
25143        storage_wrong_poll_id: bool,
25144        reject_query_protocol: bool,
25145        reject_query_completion: bool,
25146        waiting_query_worker: bool,
25147        decline_registration: bool,
25148        complete_named_signal: bool,
25149        poll_failures_per_path: usize,
25150        long_poll_capacity_responses_per_path: usize,
25151        heartbeat_failures: usize,
25152        heartbeat_failure_request: Option<usize>,
25153        delayed_heartbeat_request: Option<usize>,
25154        heartbeat_response_delay: Duration,
25155        concurrent_requests: bool,
25156        unauthorized_polls: bool,
25157        reject_registration: bool,
25158        reject_registration_protocol: bool,
25159        reject_deregistration: bool,
25160        reject_deregistration_protocol: bool,
25161        cancelled_activity: bool,
25162        draining_polls: bool,
25163        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
25164        workflow_completion_status: Option<&'static str>,
25165        workflow_completion_body: Option<&'static str>,
25166    }
25167
25168    impl MockWorkerServer {
25169        fn start() -> Self {
25170            Self::start_with_behavior(MockWorkerBehavior::default())
25171        }
25172
25173        fn reject_query_protocol() -> Self {
25174            Self::start_with_behavior(MockWorkerBehavior {
25175                reject_query_protocol: true,
25176                ..MockWorkerBehavior::default()
25177            })
25178        }
25179
25180        fn reject_query_completion() -> Self {
25181            Self::start_with_behavior(MockWorkerBehavior {
25182                reject_query_completion: true,
25183                ..MockWorkerBehavior::default()
25184            })
25185        }
25186
25187        fn waiting_query_worker() -> Self {
25188            Self::start_with_behavior(MockWorkerBehavior {
25189                waiting_query_worker: true,
25190                complete_named_signal: true,
25191                ..MockWorkerBehavior::default()
25192            })
25193        }
25194
25195        fn transient_worker_failures() -> Self {
25196            Self::start_with_behavior(MockWorkerBehavior {
25197                poll_failures_per_path: 1,
25198                heartbeat_failures: 1,
25199                ..MockWorkerBehavior::default()
25200            })
25201        }
25202
25203        fn consecutive_poll_failures(count: usize) -> Self {
25204            Self::start_with_behavior(MockWorkerBehavior {
25205                poll_failures_per_path: count,
25206                ..MockWorkerBehavior::default()
25207            })
25208        }
25209
25210        fn capacity_limited_activity_poll() -> Self {
25211            Self::start_with_behavior(MockWorkerBehavior {
25212                long_poll_capacity_responses_per_path: 1,
25213                ..MockWorkerBehavior::default()
25214            })
25215        }
25216
25217        fn delayed_heartbeat_worker() -> Self {
25218            Self::start_with_behavior(MockWorkerBehavior {
25219                waiting_query_worker: true,
25220                delayed_heartbeat_request: Some(2),
25221                heartbeat_response_delay: Duration::from_millis(1_500),
25222                concurrent_requests: true,
25223                cancelled_activity: true,
25224                ..MockWorkerBehavior::default()
25225            })
25226        }
25227
25228        fn heartbeat_retry_worker() -> Self {
25229            Self::start_with_behavior(MockWorkerBehavior {
25230                waiting_query_worker: true,
25231                heartbeat_failure_request: Some(2),
25232                concurrent_requests: true,
25233                ..MockWorkerBehavior::default()
25234            })
25235        }
25236
25237        fn unavailable_polls() -> Self {
25238            Self::start_with_behavior(MockWorkerBehavior {
25239                poll_failures_per_path: usize::MAX,
25240                ..MockWorkerBehavior::default()
25241            })
25242        }
25243
25244        fn unauthorized_polls() -> Self {
25245            Self::start_with_behavior(MockWorkerBehavior {
25246                unauthorized_polls: true,
25247                ..MockWorkerBehavior::default()
25248            })
25249        }
25250
25251        fn rejected_registration() -> Self {
25252            Self::start_with_behavior(MockWorkerBehavior {
25253                reject_registration: true,
25254                ..MockWorkerBehavior::default()
25255            })
25256        }
25257
25258        fn rejected_registration_protocol() -> Self {
25259            Self::start_with_behavior(MockWorkerBehavior {
25260                reject_registration_protocol: true,
25261                ..MockWorkerBehavior::default()
25262            })
25263        }
25264
25265        fn declined_registration() -> Self {
25266            Self::start_with_behavior(MockWorkerBehavior {
25267                decline_registration: true,
25268                ..MockWorkerBehavior::default()
25269            })
25270        }
25271
25272        fn rejected_deregistration() -> Self {
25273            Self::start_with_behavior(MockWorkerBehavior {
25274                reject_deregistration: true,
25275                ..MockWorkerBehavior::default()
25276            })
25277        }
25278
25279        fn rejected_deregistration_protocol() -> Self {
25280            Self::start_with_behavior(MockWorkerBehavior {
25281                reject_deregistration_protocol: true,
25282                ..MockWorkerBehavior::default()
25283            })
25284        }
25285
25286        fn unauthorized_polls_and_rejected_deregistration() -> Self {
25287            Self::start_with_behavior(MockWorkerBehavior {
25288                unauthorized_polls: true,
25289                reject_deregistration: true,
25290                ..MockWorkerBehavior::default()
25291            })
25292        }
25293
25294        fn cancelled_activity() -> Self {
25295            Self::start_with_behavior(MockWorkerBehavior {
25296                cancelled_activity: true,
25297                ..MockWorkerBehavior::default()
25298            })
25299        }
25300
25301        fn draining_polls() -> Self {
25302            Self::start_with_behavior(MockWorkerBehavior {
25303                draining_polls: true,
25304                ..MockWorkerBehavior::default()
25305            })
25306        }
25307
25308        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
25309            Self::start_with_behavior(MockWorkerBehavior {
25310                invalid_task_payload_codec: Some(codec),
25311                ..MockWorkerBehavior::default()
25312            })
25313        }
25314
25315        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
25316            Self::start_with_behavior(MockWorkerBehavior {
25317                workflow_completion_status: Some(status),
25318                workflow_completion_body: Some(body),
25319                ..MockWorkerBehavior::default()
25320            })
25321        }
25322
25323        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
25324            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
25325            listener
25326                .set_nonblocking(true)
25327                .expect("configure mock listener");
25328            let addr = listener.local_addr().expect("mock server address");
25329            let stop = Arc::new(AtomicBool::new(false));
25330            let server_stop = Arc::clone(&stop);
25331            let requests = Arc::new(Mutex::new(Vec::new()));
25332            let server_requests = Arc::clone(&requests);
25333            let thread = thread::spawn(move || {
25334                let mut request_threads = Vec::new();
25335                while !server_stop.load(Ordering::SeqCst) {
25336                    match listener.accept() {
25337                        Ok((mut stream, _)) => {
25338                            if behavior.concurrent_requests {
25339                                let requests = Arc::clone(&server_requests);
25340                                request_threads.push(thread::spawn(move || {
25341                                    handle_mock_worker_request(&mut stream, &requests, behavior)
25342                                }));
25343                            } else {
25344                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
25345                            }
25346                        }
25347                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
25348                            let mut index = 0;
25349                            while index < request_threads.len() {
25350                                if request_threads[index].is_finished() {
25351                                    request_threads
25352                                        .swap_remove(index)
25353                                        .join()
25354                                        .expect("join mock request");
25355                                } else {
25356                                    index += 1;
25357                                }
25358                            }
25359                            thread::sleep(Duration::from_millis(5));
25360                        }
25361                        Err(_) => break,
25362                    }
25363                }
25364                for request_thread in request_threads {
25365                    request_thread.join().expect("join mock request");
25366                }
25367            });
25368
25369            Self {
25370                addr,
25371                stop,
25372                requests,
25373                thread: Some(thread),
25374            }
25375        }
25376
25377        fn base_url(&self) -> String {
25378            format!("http://{}", self.addr)
25379        }
25380
25381        fn worker_protocol_for(&self, path: &str) -> Option<String> {
25382            self.requests
25383                .lock()
25384                .expect("captured requests")
25385                .iter()
25386                .find(|request| request.path == path)
25387                .and_then(|request| request.worker_protocol.clone())
25388        }
25389
25390        fn control_protocol_for(&self, path: &str) -> Option<String> {
25391            self.requests
25392                .lock()
25393                .expect("captured requests")
25394                .iter()
25395                .find(|request| request.path == path)
25396                .and_then(|request| request.control_protocol.clone())
25397        }
25398
25399        fn method_for(&self, path: &str) -> Option<String> {
25400            self.requests
25401                .lock()
25402                .expect("captured requests")
25403                .iter()
25404                .find(|request| request.path == path)
25405                .map(|request| request.method.clone())
25406        }
25407
25408        fn authorization_for(&self, path: &str) -> Option<String> {
25409            self.requests
25410                .lock()
25411                .expect("captured requests")
25412                .iter()
25413                .find(|request| request.path == path)
25414                .and_then(|request| request.authorization.clone())
25415        }
25416
25417        fn namespace_for(&self, path: &str) -> Option<String> {
25418            self.requests
25419                .lock()
25420                .expect("captured requests")
25421                .iter()
25422                .find(|request| request.path == path)
25423                .and_then(|request| request.namespace.clone())
25424        }
25425
25426        fn request_count(&self, path: &str) -> usize {
25427            self.requests
25428                .lock()
25429                .expect("captured requests")
25430                .iter()
25431                .filter(|request| request.path == path)
25432                .count()
25433        }
25434
25435        fn captured_paths(&self) -> Vec<String> {
25436            self.requests
25437                .lock()
25438                .expect("captured requests")
25439                .iter()
25440                .map(|request| request.path.clone())
25441                .collect()
25442        }
25443
25444        fn request_times(&self, path: &str) -> Vec<Instant> {
25445            self.requests
25446                .lock()
25447                .expect("captured requests")
25448                .iter()
25449                .filter(|request| request.path == path)
25450                .map(|request| request.received_at)
25451                .collect()
25452        }
25453
25454        fn request_body(&self, path: &str) -> Value {
25455            let requests = self.requests.lock().expect("captured requests");
25456            let body = &requests
25457                .iter()
25458                .find(|request| request.path == path)
25459                .unwrap_or_else(|| panic!("missing request for {path}"))
25460                .body;
25461            serde_json::from_str(body).unwrap_or_else(|error| {
25462                panic!("invalid JSON request body for {path}: {error}: {body:?}")
25463            })
25464        }
25465
25466        fn request_bodies(&self, path: &str) -> Vec<Value> {
25467            self.requests
25468                .lock()
25469                .expect("captured requests")
25470                .iter()
25471                .filter(|request| request.path == path)
25472                .map(|request| {
25473                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
25474                        panic!(
25475                            "invalid JSON request body for {path}: {error}: {:?}",
25476                            request.body
25477                        )
25478                    })
25479                })
25480                .collect()
25481        }
25482    }
25483
25484    impl Drop for MockWorkerServer {
25485        fn drop(&mut self) {
25486            self.stop.store(true, Ordering::SeqCst);
25487            let _ = TcpStream::connect(self.addr);
25488
25489            if let Some(thread) = self.thread.take() {
25490                thread.join().expect("join mock server");
25491            }
25492        }
25493    }
25494
25495    fn handle_mock_worker_request(
25496        stream: &mut TcpStream,
25497        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
25498        behavior: MockWorkerBehavior,
25499    ) {
25500        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
25501        let mut buffer = [0_u8; 8192];
25502        let mut request = Vec::new();
25503
25504        loop {
25505            match stream.read(&mut buffer) {
25506                Ok(0) => break,
25507                Ok(read) => {
25508                    request.extend_from_slice(&buffer[..read]);
25509                    if mock_request_is_complete(&request) {
25510                        break;
25511                    }
25512                }
25513                Err(error)
25514                    if matches!(
25515                        error.kind(),
25516                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
25517                    ) =>
25518                {
25519                    break;
25520                }
25521                Err(_) => return,
25522            }
25523        }
25524
25525        let request = String::from_utf8_lossy(&request);
25526        let body = request
25527            .split_once("\r\n\r\n")
25528            .map(|(_, body)| body)
25529            .unwrap_or_default();
25530        let path = request
25531            .lines()
25532            .next()
25533            .and_then(|line| line.split_whitespace().nth(1))
25534            .unwrap_or_default();
25535        let method = request
25536            .lines()
25537            .next()
25538            .and_then(|line| line.split_whitespace().next())
25539            .unwrap_or_default();
25540        let authorization = request.lines().find_map(|line| {
25541            let (name, value) = line.split_once(':')?;
25542            name.eq_ignore_ascii_case("Authorization")
25543                .then(|| value.trim().to_string())
25544        });
25545        let namespace = request.lines().find_map(|line| {
25546            let (name, value) = line.split_once(':')?;
25547            name.eq_ignore_ascii_case("X-Namespace")
25548                .then(|| value.trim().to_string())
25549        });
25550        let worker_protocol = request.lines().find_map(|line| {
25551            let (name, value) = line.split_once(':')?;
25552            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
25553                .then(|| value.trim().to_string())
25554        });
25555        let control_protocol = request.lines().find_map(|line| {
25556            let (name, value) = line.split_once(':')?;
25557            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
25558                .then(|| value.trim().to_string())
25559        });
25560        let request_number = {
25561            let mut requests = requests.lock().expect("captured requests");
25562            requests.push(CapturedRequest {
25563                headers: request
25564                    .split_once("\r\n\r\n")
25565                    .map_or("", |(headers, _)| headers)
25566                    .to_owned(),
25567                method: method.to_string(),
25568                path: path.to_string(),
25569                authorization,
25570                namespace,
25571                worker_protocol: worker_protocol.clone(),
25572                control_protocol,
25573                body: body.to_string(),
25574                received_at: Instant::now(),
25575            });
25576            requests
25577                .iter()
25578                .filter(|request| request.path == path)
25579                .count()
25580        };
25581
25582        if let Some(response) = behavior
25583            .request_override
25584            .and_then(|handler| handler(path, body, request_number))
25585        {
25586            write_mock_response(stream, response.0, &response.1);
25587            return;
25588        }
25589        if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
25590            write_mock_response(stream, response.0, &response.1);
25591            return;
25592        }
25593        if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
25594            return;
25595        }
25596        let pressure_path = behavior
25597            .storage_path
25598            .is_some_and(|part| path.contains(part));
25599        let prior_failures = if path.ends_with("/poll") {
25600            behavior.poll_failures_per_path
25601        } else {
25602            0
25603        };
25604        if pressure_path
25605            && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
25606        {
25607            let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
25608            let poll_id = path
25609                .ends_with("/poll")
25610                .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
25611            let mut refusal = storage_refusal(
25612                poll_id,
25613                behavior.storage_unavailable,
25614                behavior.storage_mid_poll,
25615            );
25616            if behavior.storage_wrong_poll_id {
25617                refusal["poll_request_id"] = json!("wrong-poll");
25618            }
25619            write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
25620            return;
25621        }
25622        if path.contains("/storage-task/")
25623            || path.contains("/storage-activity/")
25624            || path.contains("/storage-query/")
25625        {
25626            write_mock_response(stream, "200 OK", "{}");
25627            return;
25628        }
25629        if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
25630            write_mock_response(stream, "200 OK", &json!({"task":{
25631                "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
25632                "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
25633                "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25634                "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25635                "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
25636            }}).to_string());
25637            return;
25638        }
25639        if behavior.storage_activity
25640            && path == "/api/worker/activity-tasks/poll"
25641            && request_number == 1
25642        {
25643            write_mock_response(stream, "200 OK", &json!({"task":{
25644                "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
25645                "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25646                "attempt_number":7, "lease_owner":"storage-worker"
25647            }}).to_string());
25648            return;
25649        }
25650
25651        if path == "/api/worker/register" {
25652            if behavior.reject_registration_protocol {
25653                write_mock_response(
25654                    stream,
25655                    "400 Bad Request",
25656                    r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
25657                );
25658                return;
25659            }
25660            if behavior.reject_registration {
25661                write_mock_response(
25662                    stream,
25663                    "503 Service Unavailable",
25664                    r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
25665                );
25666                return;
25667            }
25668        }
25669
25670        if path.starts_with("/api/worker/registrations/") {
25671            if behavior.reject_deregistration_protocol {
25672                write_mock_response(
25673                    stream,
25674                    "400 Bad Request",
25675                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
25676                );
25677            } else if behavior.reject_deregistration {
25678                write_mock_response(
25679                    stream,
25680                    "403 Forbidden",
25681                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
25682                );
25683            } else {
25684                write_mock_response(
25685                    stream,
25686                    "200 OK",
25687                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
25688                );
25689            }
25690            return;
25691        }
25692
25693        let is_poll = matches!(
25694            path,
25695            "/api/worker/workflow-tasks/poll"
25696                | "/api/worker/activity-tasks/poll"
25697                | "/api/worker/query-tasks/poll"
25698        );
25699        if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
25700            write_mock_response(
25701                stream,
25702                "429 Too Many Requests",
25703                r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
25704            );
25705            return;
25706        }
25707        if is_poll && request_number <= behavior.poll_failures_per_path {
25708            return;
25709        }
25710        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
25711            return;
25712        }
25713        if path == "/api/worker/heartbeat"
25714            && behavior.heartbeat_failure_request == Some(request_number)
25715        {
25716            return;
25717        }
25718        if path == "/api/worker/heartbeat"
25719            && behavior.delayed_heartbeat_request == Some(request_number)
25720        {
25721            thread::sleep(behavior.heartbeat_response_delay);
25722        }
25723        if behavior.unauthorized_polls && is_poll {
25724            write_mock_response(
25725                stream,
25726                "401 Unauthorized",
25727                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
25728            );
25729            return;
25730        }
25731        if behavior.draining_polls && is_poll {
25732            write_mock_response(
25733                stream,
25734                "409 Conflict",
25735                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
25736            );
25737            return;
25738        }
25739
25740        if let Some(codec_case) = behavior.invalid_task_payload_codec {
25741            if is_poll && request_number == 1 {
25742                let mut task = match path {
25743                    "/api/worker/workflow-tasks/poll" => json!({
25744                        "task_id": "codec-workflow",
25745                        "workflow_type": "codec.workflow",
25746                        "payload_codec": DEFAULT_CODEC,
25747                        "workflow_task_attempt": 1,
25748                        "lease_owner": "codec-worker"
25749                    }),
25750                    "/api/worker/activity-tasks/poll" => json!({
25751                        "task_id": "codec-activity",
25752                        "activity_attempt_id": "codec-activity-attempt",
25753                        "activity_type": "codec.activity",
25754                        "payload_codec": DEFAULT_CODEC,
25755                        "attempt_number": 1,
25756                        "lease_owner": "codec-worker"
25757                    }),
25758                    "/api/worker/query-tasks/poll" => json!({
25759                        "query_task_id": "codec-query",
25760                        "query_task_attempt": 1,
25761                        "workflow_type": "codec.workflow",
25762                        "query_name": "known",
25763                        "payload_codec": DEFAULT_CODEC,
25764                        "lease_owner": "codec-worker"
25765                    }),
25766                    _ => unreachable!("is_poll limits task codec probe paths"),
25767                };
25768                codec_case.apply(&mut task);
25769                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
25770                return;
25771            }
25772
25773            if matches!(
25774                path,
25775                "/api/worker/workflow-tasks/codec-workflow/fail"
25776                    | "/api/worker/activity-tasks/codec-activity/fail"
25777                    | "/api/worker/query-tasks/codec-query/fail"
25778            ) {
25779                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
25780                return;
25781            }
25782        }
25783
25784        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
25785            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
25786            let body = format!(
25787                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
25788            );
25789            write_mock_response(stream, "400 Bad Request", &body);
25790            return;
25791        }
25792
25793        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
25794        {
25795            write_mock_response(
25796                stream,
25797                "409 Conflict",
25798                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
25799            );
25800            return;
25801        }
25802
25803        if behavior.workflow_completion_status.is_some()
25804            && path == "/api/worker/workflow-tasks/poll"
25805            && request_number == 1
25806        {
25807            write_mock_response(
25808                stream,
25809                "200 OK",
25810                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
25811            );
25812            return;
25813        }
25814
25815        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
25816            if let (Some(status), Some(body)) = (
25817                behavior.workflow_completion_status,
25818                behavior.workflow_completion_body,
25819            ) {
25820                write_mock_response(stream, status, body);
25821                return;
25822            }
25823        }
25824
25825        if behavior.waiting_query_worker {
25826            if behavior.complete_named_signal
25827                && path == "/api/worker/workflow-tasks/poll"
25828                && request_number == 1
25829            {
25830                let body = json!({
25831                    "task": {
25832                        "task_id": "snapshot-open",
25833                        "workflow_id": "snapshot-1",
25834                        "run_id": "snapshot-run-1",
25835                        "workflow_type": "snapshot",
25836                        "payload_codec": DEFAULT_CODEC,
25837                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25838                            .expect("Avro workflow arguments"),
25839                        "history_events": [],
25840                        "workflow_task_attempt": 1,
25841                        "lease_owner": "rust-snapshot-worker"
25842                    }
25843                })
25844                .to_string();
25845                write_mock_response(stream, "200 OK", &body);
25846                return;
25847            }
25848
25849            let signal_request = request_number - usize::from(behavior.complete_named_signal);
25850            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25851            if path == "/api/worker/workflow-tasks/poll"
25852                && signal_request >= 1
25853                && signal_request <= signal_request_limit
25854            {
25855                let finish = behavior.complete_named_signal && signal_request == 3;
25856                let amounts = if signal_request == 1 {
25857                    vec![3]
25858                } else {
25859                    vec![3, 5]
25860                };
25861                let task_id = if signal_request == 1 {
25862                    "snapshot-wait-3"
25863                } else if finish {
25864                    "snapshot-finish"
25865                } else {
25866                    "snapshot-wait-5"
25867                };
25868                let mut history_events = std::iter::once(json!({
25869                    "event_type": "SignalWaitOpened",
25870                    "payload": {"sequence": 1, "signal_name": "finish"}
25871                }))
25872                .chain(amounts.iter().enumerate().map(|(index, amount)| {
25873                    json!({
25874                        "event_type": "SignalReceived",
25875                        "payload": {
25876                            "signal_id": format!("increment-{amount}"),
25877                            "signal_name": "increment",
25878                            "workflow_sequence": index + 2,
25879                            "payload_codec": DEFAULT_CODEC,
25880                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25881                                .expect("Avro signal envelope")
25882                        }
25883                    })
25884                }))
25885                .collect::<Vec<_>>();
25886                let (resume_id, resume_name, resume_arguments) = if finish {
25887                    history_events.push(json!({
25888                        "event_type": "SignalReceived",
25889                        "payload": {
25890                            "signal_id": "finish",
25891                            "signal_name": "finish",
25892                            "workflow_sequence": 4,
25893                            "payload_codec": DEFAULT_CODEC,
25894                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25895                                .expect("Avro finish signal envelope")
25896                        }
25897                    }));
25898                    (
25899                        "finish".to_string(),
25900                        "finish".to_string(),
25901                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
25902                            .expect("Avro finish resume signal"),
25903                    )
25904                } else {
25905                    let amount = amounts.last().expect("amount");
25906                    (
25907                        format!("increment-{amount}"),
25908                        "increment".to_string(),
25909                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25910                            .expect("Avro increment resume signal"),
25911                    )
25912                };
25913                let body = json!({
25914                    "task": {
25915                        "task_id": task_id,
25916                        "workflow_id": "snapshot-1",
25917                        "run_id": "snapshot-run-1",
25918                        "workflow_type": "snapshot",
25919                        "payload_codec": DEFAULT_CODEC,
25920                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25921                            .expect("Avro workflow arguments"),
25922                        "history_events": history_events,
25923                        "workflow_task_attempt": 1,
25924                        "workflow_signal_id": resume_id,
25925                        "signal_name": resume_name,
25926                        "signal_arguments": resume_arguments,
25927                        "lease_owner": "rust-snapshot-worker"
25928                    }
25929                })
25930                .to_string();
25931                write_mock_response(stream, "200 OK", &body);
25932                return;
25933            }
25934
25935            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25936                let history_events = [3, 5]
25937                    .into_iter()
25938                    .enumerate()
25939                    .map(|(index, amount)| {
25940                        json!({
25941                            "event_type": "SignalReceived",
25942                            "payload": {
25943                                "signal_id": format!("increment-{amount}"),
25944                                "signal_name": "increment",
25945                                "workflow_sequence": index + 2,
25946                                "payload_codec": DEFAULT_CODEC,
25947                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25948                                    .expect("Avro query signal envelope")
25949                            }
25950                        })
25951                    })
25952                    .collect::<Vec<_>>();
25953                let body = json!({
25954                    "task": {
25955                        "query_task_id": "snapshot-current",
25956                        "query_task_attempt": 1,
25957                        "lease_owner": "rust-snapshot-worker",
25958                        "workflow_id": "snapshot-1",
25959                        "run_id": "snapshot-run-1",
25960                        "workflow_type": "snapshot",
25961                        "query_name": "current",
25962                        "payload_codec": DEFAULT_CODEC,
25963                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25964                            .expect("Avro workflow arguments"),
25965                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25966                            .expect("Avro query arguments"),
25967                        "history_events": history_events,
25968                        "run_status": "waiting"
25969                    }
25970                })
25971                .to_string();
25972                write_mock_response(stream, "200 OK", &body);
25973                return;
25974            }
25975
25976            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25977                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25978            {
25979                write_mock_response(
25980                    stream,
25981                    "200 OK",
25982                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
25983                );
25984                return;
25985            }
25986
25987            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25988                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25989                return;
25990            }
25991
25992            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25993                write_mock_response(
25994                    stream,
25995                    "200 OK",
25996                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25997                );
25998                return;
25999            }
26000
26001            if path == "/api/worker/query-tasks/snapshot-current/complete" {
26002                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
26003                return;
26004            }
26005        }
26006
26007        if matches!(
26008            path,
26009            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
26010        ) {
26011            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
26012                .expect("typed mock result");
26013            let body = json!({
26014                "result": typed_fidelity_probe().into_json().expect("result projection"),
26015                "result_envelope": result,
26016            })
26017            .to_string();
26018            write_mock_response(stream, "200 OK", &body);
26019            return;
26020        }
26021
26022        if path == "/api/workflows/typed-1" {
26023            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
26024                .expect("typed mock result");
26025            let body = json!({
26026                "workflow_id": "typed-1",
26027                "run_id": "run-typed-1",
26028                "workflow_type": "typed.echo",
26029                "status": "completed",
26030                "output": typed_fidelity_probe().into_json().expect("output projection"),
26031                "output_envelope": result,
26032            })
26033            .to_string();
26034            write_mock_response(stream, "200 OK", &body);
26035            return;
26036        }
26037
26038        let (status, body) = match path {
26039            "/api/cluster/info" => ("200 OK", r#"{"limits":{"max_payload_bytes":2097152}}"#),
26040            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
26041            "/api/workflows" => (
26042                "201 Created",
26043                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
26044            ),
26045            "/api/worker/register" if behavior.decline_registration => (
26046                "200 OK",
26047                r#"{"worker_id":"declined-worker","registered":false}"#,
26048            ),
26049            "/api/worker/register" if behavior.waiting_query_worker => (
26050                "200 OK",
26051                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
26052            ),
26053            "/api/worker/register" => (
26054                "200 OK",
26055                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
26056            ),
26057            "/api/worker/heartbeat" => ("200 OK", "{}"),
26058            "/api/worker/activity-tasks/poll"
26059                if behavior.cancelled_activity && request_number == 1 =>
26060            {
26061                (
26062                    "200 OK",
26063                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
26064                )
26065            }
26066            "/api/worker/activity-tasks/poll"
26067                if behavior.long_poll_capacity_responses_per_path > 0
26068                    && request_number
26069                        == behavior
26070                            .long_poll_capacity_responses_per_path
26071                            .saturating_add(1) =>
26072            {
26073                (
26074                    "200 OK",
26075                    r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
26076                )
26077            }
26078            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
26079                ("200 OK", r#"{"task":null}"#)
26080            }
26081            "/api/worker/query-tasks/poll"
26082                if behavior.reject_query_completion && request_number == 1 =>
26083            {
26084                (
26085                    "200 OK",
26086                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
26087                )
26088            }
26089            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
26090            "/api/worker/query-tasks/query-capture/complete"
26091            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
26092            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
26093                "200 OK",
26094                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
26095            ),
26096            "/api/worker/activity-tasks/activity-cancel/complete" => (
26097                "409 Conflict",
26098                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
26099            ),
26100            "/api/worker/activity-tasks/activity-typed/complete"
26101            | "/api/worker/activity-tasks/activity-typed/fail"
26102            | "/api/worker/activity-tasks/capacity-activity/complete"
26103            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
26104            "/api/workflows/counter-1/query/current" => (
26105                "200 OK",
26106                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
26107            ),
26108            "/api/workflows/counter-1/query/missing" => (
26109                "404 Not Found",
26110                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
26111            ),
26112            "/api/workflows/wf-lifecycle/cancel" => (
26113                "200 OK",
26114                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
26115            ),
26116            "/api/workflows/wf-lifecycle/terminate" => (
26117                "200 OK",
26118                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
26119            ),
26120            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
26121                "200 OK",
26122                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
26123            ),
26124            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
26125                "200 OK",
26126                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
26127            ),
26128            "/api/workflows/wf-lifecycle/runs/run-failed/redrive" => (
26129                "202 Accepted",
26130                r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
26131            ),
26132            "/api/workflows/wf-lifecycle/runs/run-failed-existing/redrive" => (
26133                "200 OK",
26134                r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed-existing","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
26135            ),
26136            "/api/workflows/wf-lifecycle/runs/run-completed/redrive" => (
26137                "409 Conflict",
26138                r#"{"workflow_id":"wf-lifecycle","run_id":"run-completed","reason":"run_not_failed","message":"Run is not failed."}"#,
26139            ),
26140            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
26141            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
26142                "409 Conflict",
26143                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
26144            ),
26145            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
26146                "200 OK",
26147                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
26148            ),
26149            "/api/workflows/wf-cancelled" => (
26150                "200 OK",
26151                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
26152            ),
26153            "/api/workflows/wf-terminated" => (
26154                "200 OK",
26155                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
26156            ),
26157            "/api/workflows/wf-timed-out" => (
26158                "200 OK",
26159                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
26160            ),
26161            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
26162                "200 OK",
26163                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
26164            ),
26165            "/api/workflows/wf-selected" => (
26166                "200 OK",
26167                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
26168            ),
26169            "/api/workflows/wf-selected/runs/run-selected" => (
26170                "200 OK",
26171                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
26172            ),
26173            _ => ("404 Not Found", r#"{"message":"not found"}"#),
26174        };
26175        write_mock_response(stream, status, body);
26176    }
26177
26178    fn mock_request_is_complete(request: &[u8]) -> bool {
26179        let Some(header_end) = request
26180            .windows(4)
26181            .position(|window| window == b"\r\n\r\n")
26182            .map(|position| position + 4)
26183        else {
26184            return false;
26185        };
26186        let headers = String::from_utf8_lossy(&request[..header_end]);
26187        let content_length = headers.lines().find_map(|line| {
26188            let (name, value) = line.split_once(':')?;
26189            name.eq_ignore_ascii_case("content-length")
26190                .then(|| value.trim().parse::<usize>().ok())
26191                .flatten()
26192        });
26193
26194        request.len() >= header_end + content_length.unwrap_or(0)
26195    }
26196
26197    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
26198        let response = format!(
26199            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
26200            body.len()
26201        );
26202
26203        let _ = stream.write_all(response.as_bytes());
26204        let _ = stream.flush();
26205    }
26206
26207    fn activity_completion_backend_recovery(
26208        path: &str,
26209        _body: &str,
26210        request_number: usize,
26211    ) -> Option<(&'static str, String)> {
26212        match path {
26213            "/api/worker/activity-tasks/poll" if request_number == 1 => Some((
26214                "200 OK",
26215                r#"{"task":{"task_id":"recover-activity","activity_attempt_id":"recover-attempt","activity_type":"recover.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"recover-worker"}}"#.to_string(),
26216            )),
26217            "/api/worker/activity-tasks/recover-activity/complete" if request_number == 1 => Some((
26218                "503 Service Unavailable",
26219                r#"{"reason":"backend_unavailable","operation":"complete_activity_task","outcome":"unknown","worker_id":"recover-worker","task_queue":null,"retryable":true,"retry_after_seconds":1,"task_id":"recover-activity","lease_owner":"recover-worker","activity_attempt_id":"recover-attempt"}"#.to_string(),
26220            )),
26221            "/api/worker/activity-tasks/recover-activity/complete" => Some((
26222                "200 OK",
26223                r#"{"outcome":"completed","recorded":true}"#.to_string(),
26224            )),
26225            _ => None,
26226        }
26227    }
26228
26229    fn workflow_completion_backend_recovery(
26230        path: &str,
26231        _body: &str,
26232        request_number: usize,
26233    ) -> Option<(&'static str, String)> {
26234        match path {
26235            "/api/worker/workflow-tasks/poll" if request_number == 1 => Some((
26236                "200 OK",
26237                include_str!("../tests/fixtures/worker-completion-recovery.json").to_string(),
26238            )),
26239            "/api/worker/workflow-tasks/recover-workflow/complete" if request_number == 1 => Some((
26240                "503 Service Unavailable",
26241                r#"{"reason":"backend_unavailable","operation":"complete_workflow_task","outcome":"unknown","worker_id":"recover-worker","task_queue":null,"retryable":true,"retry_after_seconds":1,"task_id":"recover-workflow","lease_owner":"recover-worker","workflow_task_attempt":3}"#.to_string(),
26242            )),
26243            "/api/worker/workflow-tasks/recover-workflow/complete" => Some((
26244                "200 OK",
26245                r#"{"outcome":"completed","recorded":true}"#.to_string(),
26246            )),
26247            _ => None,
26248        }
26249    }
26250
26251    fn activity_completion_stale_after_backend_recovery(
26252        path: &str,
26253        body: &str,
26254        request_number: usize,
26255    ) -> Option<(&'static str, String)> {
26256        if path == "/api/worker/activity-tasks/recover-activity/complete" && request_number > 1 {
26257            return Some((
26258                "409 Conflict",
26259                r#"{"task_id":"recover-activity","activity_attempt_id":"recover-attempt","lease_owner":"recover-worker","reason":"stale_attempt","outcome":"completed","recorded":false,"activity_status":"completed","attempt_status":"completed","task_status":"completed"}"#.to_string(),
26260            ));
26261        }
26262        activity_completion_backend_recovery(path, body, request_number)
26263    }
26264
26265    fn activity_completion_remains_unavailable(
26266        path: &str,
26267        body: &str,
26268        request_number: usize,
26269    ) -> Option<(&'static str, String)> {
26270        if path == "/api/worker/activity-tasks/recover-activity/complete" {
26271            return activity_completion_backend_recovery(path, body, 1);
26272        }
26273        activity_completion_backend_recovery(path, body, request_number)
26274    }
26275
26276    #[tokio::test]
26277    async fn managed_activity_retries_same_fenced_completion_after_backend_loss() {
26278        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
26279            request_override: Some(activity_completion_backend_recovery),
26280            ..MockWorkerBehavior::default()
26281        });
26282        let client = Client::builder(server.base_url())
26283            .timeout(Duration::from_secs(2))
26284            .build()
26285            .expect("client");
26286        let handler_calls = Arc::new(AtomicUsize::new(0));
26287        let mut worker = Worker::new(client, "recover-queue")
26288            .worker_id("recover-worker")
26289            .poll_timeout(Duration::from_millis(10));
26290        worker.register_activity("recover.activity", {
26291            let handler_calls = Arc::clone(&handler_calls);
26292            move |_ctx, _args| {
26293                let handler_calls = Arc::clone(&handler_calls);
26294                async move {
26295                    handler_calls.fetch_add(1, Ordering::SeqCst);
26296                    Ok(json!("done"))
26297                }
26298            }
26299        });
26300
26301        assert_eq!(worker.run_once().await.expect("activity settled"), 1);
26302        assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
26303        let bodies = server.request_bodies("/api/worker/activity-tasks/recover-activity/complete");
26304        assert_eq!(bodies.len(), 2);
26305        assert_eq!(bodies[0], bodies[1]);
26306    }
26307
26308    #[tokio::test]
26309    async fn managed_workflow_retries_same_fenced_completion_after_backend_loss() {
26310        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
26311            request_override: Some(workflow_completion_backend_recovery),
26312            ..MockWorkerBehavior::default()
26313        });
26314        let client = Client::builder(server.base_url())
26315            .timeout(Duration::from_secs(2))
26316            .build()
26317            .expect("client");
26318        let handler_calls = Arc::new(AtomicUsize::new(0));
26319        let mut worker = Worker::new(client, "recover-queue")
26320            .worker_id("recover-worker")
26321            .poll_timeout(Duration::from_millis(10));
26322        worker.register_workflow("recover.workflow", {
26323            let handler_calls = Arc::clone(&handler_calls);
26324            move |_ctx, _args| {
26325                let handler_calls = Arc::clone(&handler_calls);
26326                async move {
26327                    handler_calls.fetch_add(1, Ordering::SeqCst);
26328                    Ok(json!("done"))
26329                }
26330            }
26331        });
26332
26333        assert_eq!(worker.run_once().await.expect("workflow settled"), 1);
26334        assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
26335        let bodies = server.request_bodies("/api/worker/workflow-tasks/recover-workflow/complete");
26336        assert_eq!(bodies.len(), 2);
26337        assert_eq!(bodies[0], bodies[1]);
26338    }
26339
26340    #[tokio::test]
26341    async fn managed_activity_accepts_authoritative_stale_outcome_after_unknown_completion() {
26342        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
26343            request_override: Some(activity_completion_stale_after_backend_recovery),
26344            ..MockWorkerBehavior::default()
26345        });
26346        let client = Client::builder(server.base_url())
26347            .timeout(Duration::from_secs(2))
26348            .build()
26349            .expect("client");
26350        let handler_calls = Arc::new(AtomicUsize::new(0));
26351        let mut worker = Worker::new(client, "recover-queue")
26352            .worker_id("recover-worker")
26353            .poll_timeout(Duration::from_millis(10));
26354        worker.register_activity("recover.activity", {
26355            let handler_calls = Arc::clone(&handler_calls);
26356            move |_ctx, _args| {
26357                let handler_calls = Arc::clone(&handler_calls);
26358                async move {
26359                    handler_calls.fetch_add(1, Ordering::SeqCst);
26360                    Ok(json!("done"))
26361                }
26362            }
26363        });
26364
26365        assert_eq!(
26366            worker
26367                .run_once()
26368                .await
26369                .expect("authoritative stale outcome"),
26370            1
26371        );
26372        assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
26373        let bodies = server.request_bodies("/api/worker/activity-tasks/recover-activity/complete");
26374        assert_eq!(bodies.len(), 2);
26375        assert_eq!(bodies[0], bodies[1]);
26376    }
26377
26378    #[test]
26379    fn fenced_completion_retry_rejects_mismatched_or_incomplete_identity() {
26380        let request = json!({"lease_owner":"recover-worker","activity_attempt_id":"recover-attempt","result":{"codec":"avro","blob":"value"}});
26381        let response = json!({
26382            "reason":"backend_unavailable", "operation":"complete_activity_task",
26383            "outcome":"unknown", "worker_id":"recover-worker", "task_queue":null,
26384            "retryable":true, "retry_after_seconds":1, "task_id":"recover-activity",
26385            "lease_owner":"recover-worker", "activity_attempt_id":"recover-attempt"
26386        });
26387        let path = "/worker/activity-tasks/recover-activity/complete";
26388        let error = |body: Value| Error::Http {
26389            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
26390            body: body.to_string(),
26391        };
26392        assert_eq!(
26393            worker_fenced_completion_retry_after(&error(response.clone()), path, Some(&request)),
26394            Some(Duration::from_secs(1))
26395        );
26396
26397        for (field, invalid) in [
26398            ("operation", json!("poll_activity_task")),
26399            ("outcome", json!("rejected")),
26400            ("worker_id", json!("another-worker")),
26401            ("task_queue", json!("recover-queue")),
26402            ("retryable", json!(false)),
26403            ("retry_after_seconds", json!(0)),
26404            ("task_id", json!("another-task")),
26405            ("lease_owner", json!("another-worker")),
26406            ("activity_attempt_id", json!("another-attempt")),
26407        ] {
26408            let mut invalid_response = response.clone();
26409            invalid_response[field] = invalid;
26410            assert_eq!(
26411                worker_fenced_completion_retry_after(
26412                    &error(invalid_response),
26413                    path,
26414                    Some(&request)
26415                ),
26416                None,
26417                "must not retry mismatched {field}"
26418            );
26419        }
26420        assert_eq!(
26421            worker_fenced_completion_retry_after(&error(response), path, None),
26422            None
26423        );
26424    }
26425
26426    #[tokio::test]
26427    async fn shutdown_interrupts_fenced_completion_backend_wait() {
26428        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
26429            request_override: Some(activity_completion_remains_unavailable),
26430            ..MockWorkerBehavior::default()
26431        });
26432        let client = Client::builder(server.base_url())
26433            .timeout(Duration::from_secs(2))
26434            .build()
26435            .expect("client");
26436        let stop = Arc::new(AtomicBool::new(false));
26437        let worker = Worker::new(client, "recover-queue")
26438            .worker_id("recover-worker")
26439            .with_storage_admission(Arc::clone(&stop));
26440        let completion = tokio::spawn(async move {
26441            worker
26442                .client
26443                .complete_activity_task(
26444                    "recover-activity",
26445                    "recover-attempt",
26446                    "recover-worker",
26447                    json!("done"),
26448                    DEFAULT_CODEC,
26449                )
26450                .await
26451        });
26452        let path = "/api/worker/activity-tasks/recover-activity/complete";
26453        tokio::time::timeout(Duration::from_secs(2), async {
26454            while server.request_count(path) == 0 {
26455                tokio::time::sleep(Duration::from_millis(10)).await;
26456            }
26457        })
26458        .await
26459        .expect("first completion request");
26460        stop.store(true, Ordering::SeqCst);
26461        let error = tokio::time::timeout(Duration::from_secs(1), completion)
26462            .await
26463            .expect("shutdown must interrupt retry wait")
26464            .expect("completion task")
26465            .expect_err("backend is still unavailable");
26466        assert!(matches!(error, Error::ActivityTaskRejected(rejection) if rejection.status == 503));
26467        assert_eq!(server.request_count(path), 1);
26468    }
26469}