Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{type_name, Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use chrono::DateTime;
20use futures_util::{future::OptionFuture, task::noop_waker_ref};
21use serde::{
22    de::DeserializeOwned,
23    ser::{SerializeMap, SerializeSeq},
24    Deserialize, Deserializer, Serialize, Serializer,
25};
26pub use serde_json::{json, Value};
27use sha2::{Digest, Sha256};
28use thiserror::Error;
29pub use uuid::Uuid;
30
31pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
32/// First additive worker protocol that defines portable worker-affinity features.
33pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
34pub const CONTROL_PLANE_VERSION: &str = "2";
35pub const DEFAULT_CODEC: &str = "avro";
36pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
37/// Worker-registration capability for authored condition-wait occurrence identity.
38pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
39    "condition_wait_occurrence_identity";
40/// Worker-registration capability for portable memo upserts.
41pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
42/// Worker-registration capability for server-routed read-only queries.
43pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
44/// Worker-registration capability for canonical typed search attributes.
45pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
46/// Worker-registration capability for synchronous workflow updates.
47pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
48/// Worker-registration capability for durable named input streams.
49pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
50/// Worker-registration capability for persisted first-completion selection.
51pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
52pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
53pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
54pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
55pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
56pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
57/// First additive worker protocol that defines query-task transport.
58pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
59/// First additive worker protocol that defines typed search-attribute upserts.
60pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
61/// First additive worker protocol that defines portable memo upserts.
62pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
63/// First additive worker protocol that preserves declared search-attribute types.
64pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
65/// First additive worker protocol that defines external durable condition waits.
66pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
67/// First additive worker protocol that preserves authored condition-wait occurrences.
68pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
69/// First additive worker protocol that defines durable selection groups.
70pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
71
72pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
73    let Some((major, minor)) = version.split_once('.') else {
74        return false;
75    };
76    major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
77}
78
79fn validate_user_signal_name(signal_name: &str) -> Result<()> {
80    if signal_name == MESSAGE_STREAM_SIGNAL {
81        return Err(Error::Codec(format!(
82            "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
83        )));
84    }
85    Ok(())
86}
87
88const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
89const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
90    "Workflow task waiting for scheduled history.";
91const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
92const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
93const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
94const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
95const MAX_MEMO_ENTRIES: usize = 100;
96const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
97const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
98
99const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
100    "lease_expired",
101    "query_task_not_found",
102    "query_task_not_leased",
103    "query_task_timed_out",
104];
105
106/// Truthful service-worker manifest for features this SDK currently refuses.
107pub fn portable_worker_affinity_capability_manifest() -> Value {
108    json!({
109        "local_activities": {
110            "supported": false,
111            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
112            "reason": "rust_worker_does_not_execute_record_local_activity",
113        },
114        "worker_sessions": {
115            "supported": false,
116            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
117            "reason": "rust_worker_has_no_typed_session_lifecycle",
118        },
119        "sticky_execution": {
120            "supported": false,
121            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
122            "reason": "rust_worker_uses_complete_durable_history_replay",
123        },
124    })
125}
126
127/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
128pub const AVRO_VALUE_SCHEMA_JSON: &str =
129    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
130pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
131pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
132const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
133
134static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
135static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
136    OnceLock::new();
137
138#[derive(Clone, Copy)]
139enum RequestProtocol {
140    ControlPlane,
141    Worker(&'static str),
142}
143
144pub type Result<T> = std::result::Result<T, Error>;
145
146#[derive(Debug, Error)]
147pub enum Error {
148    #[error("transport error: {0}")]
149    Transport(#[from] reqwest::Error),
150    #[error(
151        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
152    )]
153    InvalidBaseUrl,
154    #[error("json error: {0}")]
155    Json(#[from] serde_json::Error),
156    #[error("http {status}: {body}")]
157    Http {
158        status: reqwest::StatusCode,
159        body: String,
160    },
161    #[error("codec error: {0}")]
162    Codec(String),
163    #[error(transparent)]
164    QueryFailed(QueryFailure),
165    #[error(transparent)]
166    Protocol(ProtocolFailure),
167    #[error(transparent)]
168    NonDeterministicReplay(ReplayFailure),
169    #[error(transparent)]
170    ChildWorkflowFailed(ChildWorkflowFailure),
171    #[error(transparent)]
172    ActivityFailed(ActivityFailure),
173    #[error(transparent)]
174    ParallelFailed(ParallelFailure),
175    #[error(transparent)]
176    SagaCompensationFailed(SagaCompensationFailure),
177    #[error(transparent)]
178    InvalidParallelGroup(ParallelGroupError),
179    #[error(transparent)]
180    DurableOperationCancelled(DurableOperationCancelled),
181    #[error(transparent)]
182    WorkflowCancellationRequested(WorkflowCancellationRequested),
183    #[error(transparent)]
184    WorkflowCommandRejected(WorkflowCommandRejection),
185    #[error(transparent)]
186    WorkflowFailed(WorkflowTerminalOutcome),
187    #[error(transparent)]
188    WorkflowCancelled(WorkflowTerminalOutcome),
189    #[error(transparent)]
190    WorkflowTerminated(WorkflowTerminalOutcome),
191    #[error(transparent)]
192    WorkflowTimedOut(WorkflowTerminalOutcome),
193    #[error(transparent)]
194    ActivityTaskRejected(ActivityTaskRejection),
195    #[error("workflow handler {0:?} is not registered")]
196    WorkflowNotRegistered(String),
197    #[error("activity handler {0:?} is not registered")]
198    ActivityNotRegistered(String),
199    #[error(
200        "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
201    )]
202    HandlerType {
203        handler_kind: HandlerKind,
204        handler_name: String,
205        value_kind: HandlerValueKind,
206        rust_type: &'static str,
207        message: String,
208    },
209    #[error("workflow future yielded without emitting a durable command")]
210    WorkflowYieldedWithoutCommand,
211    #[error(
212        "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
213    )]
214    MissingWorkflowCommandIdentity,
215    #[error("workflow state lock is poisoned")]
216    WorkflowStatePoisoned,
217    #[error("timer duration is too large for the worker protocol")]
218    TimerDurationOverflow,
219    #[error(transparent)]
220    InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
221    #[error(transparent)]
222    InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
223    #[error("operation timed out")]
224    Timeout,
225    #[error(
226        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
227    )]
228    MissingRoleCredentials {
229        role: &'static str,
230        opposite_role: &'static str,
231    },
232    #[error("worker loop error: {0}")]
233    WorkerLoop(String),
234    #[error(
235        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
236    )]
237    UnsupportedUpdateValidators { workflow_type: String },
238    #[error("{primary}; worker deregistration also failed: {deregistration}")]
239    WorkerShutdown {
240        primary: Box<Error>,
241        deregistration: Box<Error>,
242    },
243    #[error("invalid child workflow options: {0}")]
244    InvalidChildWorkflowOptions(String),
245    #[error("invalid workflow memo update: {0}")]
246    InvalidMemoUpdate(String),
247    #[error(
248        "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
249    )]
250    WorkflowMemoUpdatesUnavailable,
251    #[error(transparent)]
252    InvalidActivityOptions(ActivityOptionsError),
253    #[error(transparent)]
254    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
255    #[doc(hidden)]
256    #[error("workflow requested continue as new")]
257    ContinueAsNew(ContinueAsNewRequest),
258}
259
260/// Validation failure for a durable condition-wait definition.
261#[derive(Clone, Debug, Error, PartialEq, Eq)]
262pub enum ConditionWaitOptionsError {
263    #[error("condition_key must be non-empty")]
264    EmptyKey,
265    #[error("condition_definition_fingerprint must be non-empty")]
266    EmptyPredicateIdentity,
267    #[error("condition timeout is too large for the worker protocol")]
268    TimeoutOverflow,
269}
270
271/// Stable identity and optional durable timeout for a condition wait.
272///
273/// `predicate_identity` is recorded as the worker protocol's
274/// `condition_definition_fingerprint` and must change whenever predicate
275/// behavior changes. Prefer the [`wait_condition!`] macro when the predicate
276/// is written inline; it derives this identity from the predicate tokens.
277#[derive(Clone, Debug, PartialEq, Eq)]
278pub struct ConditionWaitOptions {
279    condition_key: String,
280    predicate_identity: String,
281    timeout: Option<Duration>,
282}
283
284impl ConditionWaitOptions {
285    pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
286        Self {
287            condition_key: condition_key.into(),
288            predicate_identity: predicate_identity.into(),
289            timeout: None,
290        }
291    }
292
293    pub fn timeout(mut self, timeout: Duration) -> Self {
294        self.timeout = Some(timeout);
295        self
296    }
297
298    fn validate(
299        &self,
300    ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
301        let condition_key = self.condition_key.trim();
302        if condition_key.is_empty() {
303            return Err(ConditionWaitOptionsError::EmptyKey);
304        }
305        let predicate_identity = self.predicate_identity.trim();
306        if predicate_identity.is_empty() {
307            return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
308        }
309        let timeout_seconds = self
310            .timeout
311            .map(|timeout| {
312                timeout
313                    .as_secs()
314                    .checked_add(u64::from(timeout.subsec_nanos() > 0))
315                    .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
316            })
317            .transpose()?;
318
319        Ok(ValidatedConditionWaitOptions {
320            condition_key: condition_key.to_string(),
321            predicate_identity: predicate_identity.to_string(),
322            timeout_seconds,
323        })
324    }
325}
326
327#[derive(Clone, Debug, PartialEq, Eq)]
328struct ValidatedConditionWaitOptions {
329    condition_key: String,
330    predicate_identity: String,
331    timeout_seconds: Option<u64>,
332}
333
334const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
335
336/// Unambiguous terminal result of a durable condition wait.
337#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
338#[serde(rename_all = "snake_case")]
339pub enum ConditionWaitResult {
340    Satisfied,
341    TimedOut,
342}
343
344impl ConditionWaitResult {
345    pub fn is_satisfied(self) -> bool {
346        self == Self::Satisfied
347    }
348
349    pub fn is_timed_out(self) -> bool {
350        self == Self::TimedOut
351    }
352}
353
354/// Build the stable condition definition identity used by [`wait_condition!`].
355#[doc(hidden)]
356pub fn __condition_definition_fingerprint(source: &str) -> String {
357    let mut digest = Sha256::new();
358    digest.update(b"durable-workflow-rust.wait-condition.v1\0");
359    digest.update(source.as_bytes());
360    format!("sha256:{:x}", digest.finalize())
361}
362
363/// Create a durable condition wait whose predicate definition is fingerprinted
364/// from its inline Rust tokens.
365///
366/// The returned [`ConditionWaitCall`] must be awaited. The timeout form is
367/// `wait_condition!(ctx, "approval", timeout: duration, || predicate)`.
368#[macro_export]
369macro_rules! wait_condition {
370    ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
371        $ctx.wait_condition(
372            $crate::ConditionWaitOptions::new(
373                $key,
374                $crate::__condition_definition_fingerprint(concat!(
375                    module_path!(),
376                    "\0",
377                    stringify!($predicate)
378                )),
379            )
380            .timeout($timeout),
381            $predicate,
382        )
383    }};
384    ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
385        $ctx.wait_condition(
386            $crate::ConditionWaitOptions::new(
387                $key,
388                $crate::__condition_definition_fingerprint(concat!(
389                    module_path!(),
390                    "\0",
391                    stringify!($predicate)
392                )),
393            ),
394            $predicate,
395        )
396    }};
397}
398
399const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
400const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
401const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
402const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
403const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
404
405/// Validation failure for a typed workflow search-attribute update.
406#[derive(Clone, Debug, Error, PartialEq, Eq)]
407pub enum SearchAttributeUpdateError {
408    #[error("search-attribute update requires at least one attribute")]
409    Empty,
410    #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
411    InvalidKey(String),
412    #[error("search-attribute update exceeds the limit of 100 attributes")]
413    TooManyAttributes,
414    #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
415    ValueTooLong {
416        key: String,
417        kind: &'static str,
418        limit: usize,
419    },
420    #[error(
421        "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
422    )]
423    EmptyString(String),
424    #[error("search attribute {0:?} has a non-finite float value")]
425    NonFiniteFloat(String),
426    #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
427    InvalidDateTime(String),
428    #[error("search-attribute update exceeds the 65536-byte protocol limit")]
429    PayloadTooLarge,
430}
431
432/// One public typed search-attribute value.
433#[derive(Clone, Debug, PartialEq)]
434pub enum SearchAttributeValue {
435    String(String),
436    Keyword(String),
437    KeywordList(Vec<String>),
438    Int(i64),
439    Float(f64),
440    Bool(bool),
441    DateTime(String),
442    Delete,
443}
444
445impl SearchAttributeValue {
446    fn type_name(&self) -> Option<&'static str> {
447        match self {
448            Self::String(_) => Some("string"),
449            Self::Keyword(_) => Some("keyword"),
450            Self::KeywordList(_) => Some("keyword_list"),
451            Self::Int(_) => Some("int"),
452            Self::Float(_) => Some("float"),
453            Self::Bool(_) => Some("bool"),
454            Self::DateTime(_) => Some("datetime"),
455            Self::Delete => None,
456        }
457    }
458
459    fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
460        let normalize_string = |value: String, kind: &'static str, limit: usize| {
461            let value = value.trim().to_string();
462            if value.is_empty() {
463                return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
464            }
465            if value.len() > limit {
466                return Err(SearchAttributeUpdateError::ValueTooLong {
467                    key: key.to_string(),
468                    kind,
469                    limit,
470                });
471            }
472            Ok(value)
473        };
474
475        match self {
476            Self::String(value) => Ok(Self::String(normalize_string(
477                value,
478                "string",
479                MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
480            )?)),
481            Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
482                value,
483                "keyword",
484                MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
485            )?)),
486            Self::KeywordList(values) => {
487                let values = values
488                    .into_iter()
489                    .map(|value| {
490                        let value = value.trim().to_string();
491                        if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
492                            return Err(SearchAttributeUpdateError::ValueTooLong {
493                                key: key.to_string(),
494                                kind: "keyword-list entry",
495                                limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
496                            });
497                        }
498                        Ok(value)
499                    })
500                    .collect::<std::result::Result<Vec<_>, _>>()?;
501                Ok(Self::KeywordList(values))
502            }
503            Self::Float(value) if !value.is_finite() => {
504                Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
505            }
506            Self::DateTime(value) => {
507                let value =
508                    normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
509                if DateTime::parse_from_rfc3339(&value).is_err() {
510                    return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
511                }
512                Ok(Self::DateTime(value))
513            }
514            value => Ok(value),
515        }
516    }
517
518    fn into_json(self) -> Value {
519        match self {
520            Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
521                Value::String(value)
522            }
523            Self::KeywordList(values) => {
524                Value::Array(values.into_iter().map(Value::String).collect())
525            }
526            Self::Int(value) => json!(value),
527            Self::Float(value) => json!(value),
528            Self::Bool(value) => json!(value),
529            Self::Delete => Value::Null,
530        }
531    }
532}
533
534/// Validated typed workflow-side search-attribute mutation.
535#[derive(Clone, Debug, Default, PartialEq)]
536pub struct SearchAttributeUpdate {
537    attributes: BTreeMap<String, SearchAttributeValue>,
538}
539
540impl SearchAttributeUpdate {
541    pub fn new() -> Self {
542        Self::default()
543    }
544
545    pub fn set(
546        mut self,
547        key: impl Into<String>,
548        value: SearchAttributeValue,
549    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
550        let key = key.into();
551        validate_search_attribute_key(&key)?;
552        if !self.attributes.contains_key(&key)
553            && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
554        {
555            return Err(SearchAttributeUpdateError::TooManyAttributes);
556        }
557        self.attributes.insert(key.clone(), value.normalized(&key)?);
558        self.validate_size()?;
559        Ok(self)
560    }
561
562    pub fn string(
563        self,
564        key: impl Into<String>,
565        value: impl Into<String>,
566    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
567        self.set(key, SearchAttributeValue::String(value.into()))
568    }
569
570    pub fn keyword(
571        self,
572        key: impl Into<String>,
573        value: impl Into<String>,
574    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
575        self.set(key, SearchAttributeValue::Keyword(value.into()))
576    }
577
578    pub fn keyword_list<I, V>(
579        self,
580        key: impl Into<String>,
581        values: I,
582    ) -> std::result::Result<Self, SearchAttributeUpdateError>
583    where
584        I: IntoIterator<Item = V>,
585        V: Into<String>,
586    {
587        self.set(
588            key,
589            SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
590        )
591    }
592
593    pub fn int(
594        self,
595        key: impl Into<String>,
596        value: i64,
597    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
598        self.set(key, SearchAttributeValue::Int(value))
599    }
600
601    pub fn float(
602        self,
603        key: impl Into<String>,
604        value: f64,
605    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
606        self.set(key, SearchAttributeValue::Float(value))
607    }
608
609    pub fn bool(
610        self,
611        key: impl Into<String>,
612        value: bool,
613    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
614        self.set(key, SearchAttributeValue::Bool(value))
615    }
616
617    pub fn datetime(
618        self,
619        key: impl Into<String>,
620        value: impl Into<String>,
621    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
622        self.set(key, SearchAttributeValue::DateTime(value.into()))
623    }
624
625    pub fn delete(
626        self,
627        key: impl Into<String>,
628    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
629        self.set(key, SearchAttributeValue::Delete)
630    }
631
632    fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
633        let (attributes, _) = self.clone().into_wire_parts();
634        if serde_json::to_vec(&attributes)
635            .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
636            .unwrap_or(true)
637        {
638            return Err(SearchAttributeUpdateError::PayloadTooLarge);
639        }
640        Ok(())
641    }
642
643    fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
644        let mut attributes = serde_json::Map::new();
645        let mut attribute_types = BTreeMap::new();
646        for (key, value) in self.attributes {
647            if let Some(type_name) = value.type_name() {
648                attribute_types.insert(key.clone(), type_name.to_string());
649            }
650            attributes.insert(key, value.into_json());
651        }
652        (Value::Object(attributes), attribute_types)
653    }
654
655    fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
656        if self.attributes.is_empty() {
657            return Err(SearchAttributeUpdateError::Empty);
658        }
659        self.validate_size()
660    }
661}
662
663fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
664    let valid = !key.is_empty()
665        && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
666        && key
667            .bytes()
668            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
669    if valid {
670        Ok(())
671    } else {
672        Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
673    }
674}
675
676/// The registered handler family reported by [`Error::HandlerType`].
677#[derive(Clone, Copy, Debug, PartialEq, Eq)]
678pub enum HandlerKind {
679    Workflow,
680    Activity,
681}
682
683impl std::fmt::Display for HandlerKind {
684    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
685        formatter.write_str(match self {
686            Self::Workflow => "workflow",
687            Self::Activity => "activity",
688        })
689    }
690}
691
692/// Whether a typed handler failed to adapt its input or result.
693#[derive(Clone, Copy, Debug, PartialEq, Eq)]
694pub enum HandlerValueKind {
695    Input,
696    Result,
697}
698
699impl std::fmt::Display for HandlerValueKind {
700    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
701        formatter.write_str(match self {
702            Self::Input => "input",
703            Self::Result => "result",
704        })
705    }
706}
707
708/// The lifecycle command sent to a workflow execution.
709#[derive(Clone, Copy, Debug, PartialEq, Eq)]
710pub enum WorkflowCommandKind {
711    Cancel,
712    Terminate,
713}
714
715impl WorkflowCommandKind {
716    fn as_str(self) -> &'static str {
717        match self {
718            Self::Cancel => "cancel",
719            Self::Terminate => "terminate",
720        }
721    }
722}
723
724/// Optional structured fields for a cancellation or termination request.
725#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
726pub struct WorkflowCommandOptions {
727    #[serde(skip_serializing_if = "Option::is_none")]
728    pub reason: Option<String>,
729    #[serde(skip_serializing_if = "Option::is_none")]
730    pub request_id: Option<String>,
731}
732
733/// Server-enforced timeout policy for a workflow start.
734///
735/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
736/// only bounds how long the caller waits. A server deadline produces a terminal
737/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
738/// `run_timeout`.
739#[derive(Clone, Debug, PartialEq, Eq)]
740pub struct WorkflowStartOptions {
741    pub execution_timeout_seconds: u64,
742    pub run_timeout_seconds: u64,
743}
744
745impl Default for WorkflowStartOptions {
746    fn default() -> Self {
747        Self {
748            execution_timeout_seconds: 3600,
749            run_timeout_seconds: 600,
750        }
751    }
752}
753
754impl WorkflowStartOptions {
755    pub fn new() -> Self {
756        Self::default()
757    }
758
759    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
760        self.execution_timeout_seconds = seconds;
761        self
762    }
763
764    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
765        self.run_timeout_seconds = seconds;
766        self
767    }
768
769    fn validate(&self) -> Result<()> {
770        if self.execution_timeout_seconds == 0 {
771            return Err(Error::Codec(
772                "execution_timeout_seconds must be at least 1".to_string(),
773            ));
774        }
775        if self.run_timeout_seconds == 0 {
776            return Err(Error::Codec(
777                "run_timeout_seconds must be at least 1".to_string(),
778            ));
779        }
780        if self.run_timeout_seconds > self.execution_timeout_seconds {
781            return Err(Error::Codec(
782                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
783            ));
784        }
785
786        Ok(())
787    }
788}
789
790/// Optional routing overrides for a continue-as-new transition.
791///
792/// Omitted values retain the current workflow type and task queue. Server-owned
793/// instance metadata is not accepted here and is carried by the server.
794#[derive(Clone, Debug, Default, PartialEq, Eq)]
795pub struct ContinueAsNewOptions {
796    pub workflow_type: Option<String>,
797    pub task_queue: Option<String>,
798}
799
800impl ContinueAsNewOptions {
801    pub fn new() -> Self {
802        Self::default()
803    }
804
805    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
806        self.workflow_type = Some(workflow_type.into());
807        self
808    }
809
810    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
811        self.task_queue = Some(task_queue.into());
812        self
813    }
814
815    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
816        for (field, value) in [
817            ("workflow_type", self.workflow_type.as_deref()),
818            ("task_queue", self.task_queue.as_deref()),
819        ] {
820            if value.is_some_and(|value| value.trim().is_empty()) {
821                return Err(ContinueAsNewOptionsError {
822                    field,
823                    message: format!("{field} must not be empty"),
824                });
825            }
826        }
827        Ok(())
828    }
829}
830
831/// A stable validation error raised before a continue-as-new command is emitted.
832#[derive(Clone, Debug, Error, PartialEq, Eq)]
833#[error("invalid continue-as-new option {field}: {message}")]
834pub struct ContinueAsNewOptionsError {
835    pub field: &'static str,
836    pub message: String,
837}
838
839/// Public history-budget information attached to the current workflow task.
840#[derive(Clone, Debug, Default, PartialEq, Eq)]
841pub struct WorkflowHistoryBudget {
842    pub event_count: u64,
843    pub size_bytes: Option<u64>,
844    pub continue_as_new_recommended: bool,
845    pub pressure: Option<String>,
846}
847
848#[doc(hidden)]
849#[derive(Clone, Debug)]
850pub struct ContinueAsNewRequest {
851    arguments: AvroValue,
852    options: ContinueAsNewOptions,
853}
854
855impl WorkflowCommandOptions {
856    pub fn new() -> Self {
857        Self::default()
858    }
859
860    pub fn reason(mut self, reason: impl Into<String>) -> Self {
861        self.reason = Some(reason.into());
862        self
863    }
864
865    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
866        self.request_id = Some(request_id.into());
867        self
868    }
869}
870
871/// The accepted, machine-readable result of a lifecycle command.
872#[derive(Clone, Debug, PartialEq)]
873pub struct WorkflowCommandResult {
874    pub command: WorkflowCommandKind,
875    pub workflow_id: String,
876    pub run_id: Option<String>,
877    pub outcome: Option<String>,
878    pub reason: Option<String>,
879    pub command_status: Option<String>,
880    pub raw: Value,
881}
882
883/// A stable rejection returned by instance- or selected-run lifecycle commands.
884#[derive(Clone, Debug, Error)]
885#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
886pub struct WorkflowCommandRejection {
887    pub command: WorkflowCommandKind,
888    pub status: u16,
889    pub reason: String,
890    pub message: String,
891    pub workflow_id: String,
892    pub run_id: Option<String>,
893    pub target_scope: Option<String>,
894    pub body: Value,
895}
896
897/// Stable terminal categories returned by [`WorkflowHandle::result`].
898#[derive(Clone, Copy, Debug, PartialEq, Eq)]
899pub enum WorkflowTerminalKind {
900    Failed,
901    Cancelled,
902    Terminated,
903    TimedOut,
904}
905
906/// A typed terminal workflow outcome with durable identity and failure metadata.
907///
908/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
909/// of parsing its display representation. Fields remain `None` when an older
910/// server did not publish that metadata.
911#[derive(Clone, Debug, Error)]
912#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
913pub struct WorkflowTerminalOutcome {
914    pub kind: WorkflowTerminalKind,
915    pub workflow_id: String,
916    pub run_id: Option<String>,
917    pub reason: String,
918    pub failure_category: Option<String>,
919    pub failure_id: Option<String>,
920    pub exception_type: Option<String>,
921    pub exception_class: Option<String>,
922    pub non_retryable: Option<bool>,
923    pub message: Option<String>,
924    pub exception: Option<Value>,
925    pub raw: Value,
926}
927
928/// A worker-side activity settlement or heartbeat rejected by durable state.
929#[derive(Clone, Debug, Error)]
930#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
931pub struct ActivityTaskRejection {
932    pub operation: String,
933    pub status: u16,
934    pub reason: String,
935    pub task_id: String,
936    pub activity_attempt_id: String,
937    pub cancel_requested: bool,
938    pub can_continue: Option<bool>,
939    pub run_closed_reason: Option<String>,
940    pub body: Value,
941}
942
943/// Stable validation categories for [`ActivityOptions`].
944#[derive(Clone, Copy, Debug, PartialEq, Eq)]
945pub enum ActivityOptionsErrorKind {
946    EmptyTaskQueue,
947    EmptyRetryPolicy,
948    InvalidMaxAttempts,
949    BackoffWithoutRetryBudget,
950    TooManyBackoffIntervals,
951    InvalidBackoffCoefficient,
952    BackoffGenerationTooLarge,
953    BackoffOverflow,
954    EmptyNonRetryableErrorType,
955    TimeoutNotPositive,
956    TimeoutOverflow,
957    TimeoutOrder,
958}
959
960/// A machine-readable activity-options validation failure.
961#[derive(Clone, Debug, Error, PartialEq, Eq)]
962#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
963pub struct ActivityOptionsError {
964    pub kind: ActivityOptionsErrorKind,
965    pub field: Option<&'static str>,
966    pub message: String,
967}
968
969impl ActivityOptionsError {
970    fn new(
971        kind: ActivityOptionsErrorKind,
972        field: Option<&'static str>,
973        message: impl Into<String>,
974    ) -> Self {
975        Self {
976            kind,
977            field,
978            message: message.into(),
979        }
980    }
981}
982
983/// Stable terminal categories returned when an awaited activity does not succeed.
984#[derive(Clone, Copy, Debug, PartialEq, Eq)]
985pub enum ActivityFailureKind {
986    Failed,
987    Cancelled,
988    TimedOut,
989}
990
991/// A stable, machine-readable terminal activity failure.
992///
993/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
994/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
995#[derive(Clone, Debug, Error)]
996#[error("activity failed ({reason}): {message}")]
997pub struct ActivityFailure {
998    pub kind: ActivityFailureKind,
999    pub reason: String,
1000    pub message: String,
1001    pub activity_execution_id: Option<String>,
1002    pub activity_attempt_id: Option<String>,
1003    pub activity_type: Option<String>,
1004    pub activity_class: Option<String>,
1005    pub attempt_number: Option<u64>,
1006    pub failure_id: Option<String>,
1007    pub failure_category: Option<String>,
1008    pub timeout_kind: Option<String>,
1009    pub non_retryable: bool,
1010    pub exception_type: Option<String>,
1011    pub exception_class: Option<String>,
1012    pub code: Option<Value>,
1013    pub exception: Option<Value>,
1014}
1015
1016/// Stable terminal categories returned when an awaited child does not succeed.
1017#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1018pub enum ChildWorkflowFailureKind {
1019    Failed,
1020    Cancelled,
1021    Terminated,
1022}
1023
1024/// A stable, machine-readable child workflow failure delivered to its parent.
1025///
1026/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
1027/// of parsing the display message. Child and parent identifiers retain the
1028/// relationship recorded in durable history across worker restarts.
1029#[derive(Clone, Debug, Error)]
1030#[error("child workflow failed ({reason}): {message}")]
1031pub struct ChildWorkflowFailure {
1032    pub kind: ChildWorkflowFailureKind,
1033    pub reason: String,
1034    pub message: String,
1035    pub parent_workflow_id: Option<String>,
1036    pub parent_workflow_run_id: Option<String>,
1037    pub child_workflow_id: Option<String>,
1038    pub child_workflow_run_id: Option<String>,
1039    pub child_workflow_type: Option<String>,
1040    pub failure_id: Option<String>,
1041    pub failure_category: Option<String>,
1042    pub exception_type: Option<String>,
1043    pub exception_class: Option<String>,
1044    pub non_retryable: bool,
1045    pub code: Option<Value>,
1046    pub exception: Option<Value>,
1047}
1048
1049/// The identity of one durable workflow execution.
1050#[derive(Clone, Debug, PartialEq, Eq)]
1051pub struct WorkflowIdentity {
1052    pub workflow_id: Option<String>,
1053    pub run_id: Option<String>,
1054}
1055
1056/// A successful child result together with its durable parent-child identity.
1057#[derive(Clone, Debug, PartialEq)]
1058pub struct ChildWorkflowResult {
1059    pub parent: WorkflowIdentity,
1060    pub child: WorkflowIdentity,
1061    pub child_workflow_type: Option<String>,
1062    pub result: Value,
1063}
1064
1065/// Lossless successful child result for fixed Avro Value workflows.
1066#[derive(Clone, Debug, PartialEq)]
1067pub struct ChildWorkflowAvroResult {
1068    pub parent: WorkflowIdentity,
1069    pub child: WorkflowIdentity,
1070    pub child_workflow_type: Option<String>,
1071    pub result: AvroValue,
1072}
1073
1074/// Stable user-facing identity for one member of a durable selection group.
1075#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1076#[serde(untagged)]
1077pub enum SelectionKey {
1078    Index(usize),
1079    Name(String),
1080}
1081
1082impl From<usize> for SelectionKey {
1083    fn from(value: usize) -> Self {
1084        Self::Index(value)
1085    }
1086}
1087
1088impl From<String> for SelectionKey {
1089    fn from(value: String) -> Self {
1090        Self::Name(value)
1091    }
1092}
1093
1094impl From<&str> for SelectionKey {
1095    fn from(value: &str) -> Self {
1096        Self::Name(value.to_string())
1097    }
1098}
1099
1100/// Typed result of explicitly awaiting a cancelled non-winning operation.
1101#[derive(Clone, Debug, Error, PartialEq, Eq)]
1102#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1103pub struct DurableOperationCancelled {
1104    pub selection_group_id: String,
1105    pub member_key: SelectionKey,
1106    pub member_index: usize,
1107    pub operation_kind: String,
1108    pub operation_identity: String,
1109}
1110
1111/// Stable identity for one enclosing deterministic parallel group.
1112///
1113/// The same fields are attached to every ordinary activity, timer, or child
1114/// workflow command in the group. Nested leaves carry an outer-to-inner path;
1115/// no Rust-specific wire command is introduced.
1116#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1117pub struct ParallelGroupMetadata {
1118    pub parallel_group_id: String,
1119    pub parallel_group_kind: String,
1120    pub parallel_group_base_sequence: u64,
1121    pub parallel_group_size: usize,
1122    pub parallel_group_index: usize,
1123    #[serde(default, skip_serializing_if = "Option::is_none")]
1124    pub parallel_group_mode: Option<String>,
1125    #[serde(default, skip_serializing_if = "Option::is_none")]
1126    pub selection_member_key: Option<SelectionKey>,
1127    #[serde(default, skip_serializing_if = "Option::is_none")]
1128    pub selection_member_index: Option<usize>,
1129    #[serde(default, skip_serializing_if = "Option::is_none")]
1130    pub selection_member_base_sequence: Option<u64>,
1131    #[serde(default, skip_serializing_if = "Option::is_none")]
1132    pub selection_member_size: Option<usize>,
1133    #[serde(default, skip_serializing_if = "Option::is_none")]
1134    pub selection_member_kind: Option<String>,
1135}
1136
1137/// One input-ordered result returned by [`WorkflowContext::parallel`].
1138#[derive(Clone, Debug, PartialEq)]
1139pub enum ParallelResult {
1140    Activity(Value),
1141    ChildWorkflow(ChildWorkflowResult),
1142    Timer,
1143    Signal(Vec<Value>),
1144    Condition(ConditionWaitResult),
1145    Group(Vec<ParallelResult>),
1146}
1147
1148/// Lossless fixed-Avro counterpart to [`ParallelResult`].
1149#[derive(Clone, Debug, PartialEq)]
1150pub enum ParallelAvroResult {
1151    Activity(AvroValue),
1152    ChildWorkflow(ChildWorkflowAvroResult),
1153    Timer,
1154    Signal(Vec<AvroValue>),
1155    Condition(ConditionWaitResult),
1156    Group(Vec<ParallelAvroResult>),
1157}
1158
1159impl ParallelAvroResult {
1160    fn into_json_result(self) -> Result<ParallelResult> {
1161        match self {
1162            Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1163            Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1164                parent: result.parent,
1165                child: result.child,
1166                child_workflow_type: result.child_workflow_type,
1167                result: result.result.into_json()?,
1168            })),
1169            Self::Timer => Ok(ParallelResult::Timer),
1170            Self::Signal(values) => Ok(ParallelResult::Signal(
1171                values
1172                    .into_iter()
1173                    .map(AvroValue::into_json)
1174                    .collect::<Result<Vec<_>>>()?,
1175            )),
1176            Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1177            Self::Group(results) => Ok(ParallelResult::Group(
1178                results
1179                    .into_iter()
1180                    .map(Self::into_json_result)
1181                    .collect::<Result<Vec<_>>>()?,
1182            )),
1183        }
1184    }
1185}
1186
1187/// One successful leaf retained when another parallel member failed.
1188#[derive(Clone, Debug, PartialEq)]
1189pub struct ParallelCompletion {
1190    pub member_path: Vec<usize>,
1191    pub result: ParallelResult,
1192}
1193
1194/// A deterministic join failed after some siblings had already completed.
1195///
1196/// `cause` retains the typed activity, child-workflow, cancellation, or codec
1197/// error. `completed` is declaration ordered and contains only durable
1198/// successes observed in the same replay. Late sibling completions can add
1199/// entries on a later replay without changing `member_path` or the selected
1200/// positional failure.
1201#[derive(Debug, Error)]
1202#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1203pub struct ParallelFailure {
1204    pub group_id: String,
1205    pub member_path: Vec<usize>,
1206    pub group_path: Vec<ParallelGroupMetadata>,
1207    pub completed: Vec<ParallelCompletion>,
1208    #[source]
1209    pub cause: Box<Error>,
1210}
1211
1212/// Stable validation error returned before an invalid group emits commands.
1213#[derive(Clone, Debug, Error, PartialEq, Eq)]
1214#[error("invalid deterministic parallel group ({reason}): {message}")]
1215pub struct ParallelGroupError {
1216    pub reason: &'static str,
1217    pub member_path: Vec<usize>,
1218    pub message: String,
1219}
1220
1221/// Cooperative workflow cancellation observed at an author-controlled point.
1222#[derive(Clone, Debug, Error, PartialEq, Eq)]
1223#[error("workflow cancellation was requested")]
1224pub struct WorkflowCancellationRequested;
1225
1226/// A forward saga failure followed by a terminal compensation failure.
1227#[derive(Debug, Error)]
1228#[error(
1229    "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1230)]
1231pub struct SagaCompensationFailure {
1232    pub initiating_failure: Box<Error>,
1233    pub compensation_failure: Box<Error>,
1234    pub compensation_activity_type: String,
1235    pub compensation_registration_order: usize,
1236}
1237
1238/// Server behavior when a parent closes while its child is still open.
1239#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1240pub enum ParentClosePolicy {
1241    #[default]
1242    Abandon,
1243    RequestCancel,
1244    Terminate,
1245}
1246
1247impl ParentClosePolicy {
1248    fn as_str(self) -> &'static str {
1249        match self {
1250            Self::Abandon => "abandon",
1251            Self::RequestCancel => "request_cancel",
1252            Self::Terminate => "terminate",
1253        }
1254    }
1255}
1256
1257/// Durable retry policy for one child workflow invocation.
1258#[derive(Clone, Debug, Default, PartialEq, Eq)]
1259pub struct ChildWorkflowRetryPolicy {
1260    pub max_attempts: Option<u32>,
1261    pub backoff_seconds: Vec<u64>,
1262    pub non_retryable_error_types: Vec<String>,
1263}
1264
1265/// Options recorded with a child-workflow command.
1266///
1267/// The task queue is mandatory so routing is explicit and replay-stable.
1268#[derive(Clone, Debug, PartialEq, Eq)]
1269pub struct ChildWorkflowOptions {
1270    pub task_queue: String,
1271    pub parent_close_policy: ParentClosePolicy,
1272    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1273    pub execution_timeout_seconds: Option<u64>,
1274    pub run_timeout_seconds: Option<u64>,
1275}
1276
1277impl ChildWorkflowOptions {
1278    pub fn new(task_queue: impl Into<String>) -> Self {
1279        Self {
1280            task_queue: task_queue.into(),
1281            parent_close_policy: ParentClosePolicy::Abandon,
1282            retry_policy: None,
1283            execution_timeout_seconds: None,
1284            run_timeout_seconds: None,
1285        }
1286    }
1287
1288    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1289        self.parent_close_policy = policy;
1290        self
1291    }
1292
1293    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1294        self.retry_policy = Some(policy);
1295        self
1296    }
1297
1298    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1299        self.execution_timeout_seconds = Some(seconds);
1300        self
1301    }
1302
1303    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1304        self.run_timeout_seconds = Some(seconds);
1305        self
1306    }
1307}
1308
1309/// Backoff intervals for one durable activity retry policy.
1310#[derive(Clone, Debug, PartialEq, Eq)]
1311pub enum ActivityBackoff {
1312    /// Use these intervals between attempts. The server repeats the final
1313    /// interval if the retry budget contains more attempts than entries.
1314    Explicit(Vec<Duration>),
1315    /// Generate one interval for every retry using integer exponential growth.
1316    Exponential {
1317        initial_interval: Duration,
1318        coefficient: u32,
1319        maximum_interval: Option<Duration>,
1320    },
1321}
1322
1323/// Durable server-side retry policy for one activity execution.
1324#[derive(Clone, Debug, Default, PartialEq, Eq)]
1325pub struct ActivityRetryPolicy {
1326    pub max_attempts: Option<u32>,
1327    pub backoff: Option<ActivityBackoff>,
1328    pub non_retryable_error_types: Vec<String>,
1329}
1330
1331impl ActivityRetryPolicy {
1332    /// Start a policy with a finite attempt budget, including the first attempt.
1333    pub fn new(max_attempts: u32) -> Self {
1334        Self {
1335            max_attempts: Some(max_attempts),
1336            ..Self::default()
1337        }
1338    }
1339
1340    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1341        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1342        self
1343    }
1344
1345    pub fn exponential_backoff(
1346        mut self,
1347        initial_interval: Duration,
1348        coefficient: u32,
1349        maximum_interval: Option<Duration>,
1350    ) -> Self {
1351        self.backoff = Some(ActivityBackoff::Exponential {
1352            initial_interval,
1353            coefficient,
1354            maximum_interval,
1355        });
1356        self
1357    }
1358
1359    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1360        self.non_retryable_error_types.push(error_type.into());
1361        self
1362    }
1363
1364    pub fn non_retryable_error_types(
1365        mut self,
1366        error_types: impl IntoIterator<Item = impl Into<String>>,
1367    ) -> Self {
1368        self.non_retryable_error_types
1369            .extend(error_types.into_iter().map(Into::into));
1370        self
1371    }
1372}
1373
1374/// Options recorded atomically on one deterministic `schedule_activity` command.
1375///
1376/// Durations are rounded up to whole seconds when encoded, so the server never
1377/// receives a shorter timeout or backoff than the caller requested.
1378#[derive(Clone, Debug, Default, PartialEq, Eq)]
1379pub struct ActivityOptions {
1380    pub task_queue: Option<String>,
1381    pub retry_policy: Option<ActivityRetryPolicy>,
1382    pub start_to_close_timeout: Option<Duration>,
1383    pub schedule_to_start_timeout: Option<Duration>,
1384    pub schedule_to_close_timeout: Option<Duration>,
1385    pub heartbeat_timeout: Option<Duration>,
1386}
1387
1388impl ActivityOptions {
1389    pub fn new() -> Self {
1390        Self::default()
1391    }
1392
1393    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1394        self.task_queue = Some(task_queue.into());
1395        self
1396    }
1397
1398    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1399        self.retry_policy = Some(policy);
1400        self
1401    }
1402
1403    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1404        self.start_to_close_timeout = Some(timeout);
1405        self
1406    }
1407
1408    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1409        self.schedule_to_start_timeout = Some(timeout);
1410        self
1411    }
1412
1413    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1414        self.schedule_to_close_timeout = Some(timeout);
1415        self
1416    }
1417
1418    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1419        self.heartbeat_timeout = Some(timeout);
1420        self
1421    }
1422
1423    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1424        if self
1425            .task_queue
1426            .as_deref()
1427            .is_some_and(|queue| queue.trim().is_empty())
1428        {
1429            return Err(ActivityOptionsError::new(
1430                ActivityOptionsErrorKind::EmptyTaskQueue,
1431                Some("task_queue"),
1432                "task_queue must not be empty",
1433            ));
1434        }
1435
1436        for (field, value) in [
1437            ("start_to_close_timeout", self.start_to_close_timeout),
1438            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1439            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1440            ("heartbeat_timeout", self.heartbeat_timeout),
1441        ] {
1442            if value.is_some_and(|value| value.is_zero()) {
1443                return Err(ActivityOptionsError::new(
1444                    ActivityOptionsErrorKind::TimeoutNotPositive,
1445                    Some(field),
1446                    format!("{field} must be positive"),
1447                ));
1448            }
1449        }
1450
1451        validate_timeout_order(
1452            "heartbeat_timeout",
1453            self.heartbeat_timeout,
1454            "start_to_close_timeout",
1455            self.start_to_close_timeout,
1456        )?;
1457        validate_timeout_order(
1458            "start_to_close_timeout",
1459            self.start_to_close_timeout,
1460            "schedule_to_close_timeout",
1461            self.schedule_to_close_timeout,
1462        )?;
1463        validate_timeout_order(
1464            "schedule_to_start_timeout",
1465            self.schedule_to_start_timeout,
1466            "schedule_to_close_timeout",
1467            self.schedule_to_close_timeout,
1468        )?;
1469
1470        Ok(ValidatedActivityOptions {
1471            task_queue: self.task_queue.clone(),
1472            retry_policy: self
1473                .retry_policy
1474                .as_ref()
1475                .map(validate_activity_retry_policy)
1476                .transpose()?,
1477            start_to_close_timeout: timeout_seconds(
1478                "start_to_close_timeout",
1479                self.start_to_close_timeout,
1480            )?,
1481            schedule_to_start_timeout: timeout_seconds(
1482                "schedule_to_start_timeout",
1483                self.schedule_to_start_timeout,
1484            )?,
1485            schedule_to_close_timeout: timeout_seconds(
1486                "schedule_to_close_timeout",
1487                self.schedule_to_close_timeout,
1488            )?,
1489            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1490        })
1491    }
1492}
1493
1494/// A deferred durable leaf or nested group for [`WorkflowContext::parallel`].
1495///
1496/// Constructors capture arguments but perform no I/O. The join validates the
1497/// complete tree, attaches the existing parallel-group metadata to every
1498/// ordinary command, schedules all leaves, and then suspends.
1499pub enum ParallelOperation {
1500    Activity {
1501        activity_type: String,
1502        options: ActivityOptions,
1503        arguments: Result<AvroValue>,
1504    },
1505    ChildWorkflow {
1506        workflow_type: String,
1507        options: ChildWorkflowOptions,
1508        arguments: Result<AvroValue>,
1509    },
1510    Timer(Duration),
1511    Signal(String),
1512    Condition {
1513        options: ConditionWaitOptions,
1514        predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1515    },
1516    Group(Vec<ParallelOperation>),
1517}
1518
1519impl ParallelOperation {
1520    pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1521        Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1522    }
1523
1524    pub fn activity_with_options<T: Serialize>(
1525        activity_type: impl Into<String>,
1526        options: ActivityOptions,
1527        args: T,
1528    ) -> Self {
1529        Self::Activity {
1530            activity_type: activity_type.into(),
1531            options,
1532            arguments: AvroValue::from_serialize(&args),
1533        }
1534    }
1535
1536    pub fn child_workflow<T: Serialize>(
1537        workflow_type: impl Into<String>,
1538        options: ChildWorkflowOptions,
1539        args: T,
1540    ) -> Self {
1541        Self::ChildWorkflow {
1542            workflow_type: workflow_type.into(),
1543            options,
1544            arguments: AvroValue::from_serialize(&args),
1545        }
1546    }
1547
1548    pub fn timer(duration: Duration) -> Self {
1549        Self::Timer(duration)
1550    }
1551
1552    pub fn signal(signal_name: impl Into<String>) -> Self {
1553        Self::Signal(signal_name.into())
1554    }
1555
1556    pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1557    where
1558        F: Fn() -> Result<bool> + Send + 'static,
1559    {
1560        Self::Condition {
1561            options,
1562            predicate: Box::new(predicate),
1563        }
1564    }
1565
1566    pub fn group(operations: Vec<ParallelOperation>) -> Self {
1567        Self::Group(operations)
1568    }
1569}
1570
1571#[derive(Clone, Debug)]
1572struct ValidatedActivityOptions {
1573    task_queue: Option<String>,
1574    retry_policy: Option<Value>,
1575    start_to_close_timeout: Option<u64>,
1576    schedule_to_start_timeout: Option<u64>,
1577    schedule_to_close_timeout: Option<u64>,
1578    heartbeat_timeout: Option<u64>,
1579}
1580
1581fn validate_timeout_order(
1582    smaller_name: &'static str,
1583    smaller: Option<Duration>,
1584    larger_name: &'static str,
1585    larger: Option<Duration>,
1586) -> std::result::Result<(), ActivityOptionsError> {
1587    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1588        return Err(ActivityOptionsError::new(
1589            ActivityOptionsErrorKind::TimeoutOrder,
1590            Some(smaller_name),
1591            format!("{smaller_name} must be <= {larger_name}"),
1592        ));
1593    }
1594    Ok(())
1595}
1596
1597fn timeout_seconds(
1598    field: &'static str,
1599    value: Option<Duration>,
1600) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1601    value
1602        .map(|value| {
1603            activity_protocol_seconds(value).ok_or_else(|| {
1604                ActivityOptionsError::new(
1605                    ActivityOptionsErrorKind::TimeoutOverflow,
1606                    Some(field),
1607                    format!("{field} is too large for the worker protocol"),
1608                )
1609            })
1610        })
1611        .transpose()
1612}
1613
1614fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1615    value
1616        .as_secs()
1617        .checked_add(u64::from(value.subsec_nanos() > 0))
1618}
1619
1620fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1621    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1622}
1623
1624fn validate_activity_retry_policy(
1625    policy: &ActivityRetryPolicy,
1626) -> std::result::Result<Value, ActivityOptionsError> {
1627    if policy.max_attempts.is_none()
1628        && policy.backoff.is_none()
1629        && policy.non_retryable_error_types.is_empty()
1630    {
1631        return Err(ActivityOptionsError::new(
1632            ActivityOptionsErrorKind::EmptyRetryPolicy,
1633            Some("retry_policy"),
1634            "retry_policy must configure at least one field",
1635        ));
1636    }
1637    if policy.max_attempts == Some(0) {
1638        return Err(ActivityOptionsError::new(
1639            ActivityOptionsErrorKind::InvalidMaxAttempts,
1640            Some("retry_policy.max_attempts"),
1641            "max_attempts must be >= 1",
1642        ));
1643    }
1644    if policy
1645        .non_retryable_error_types
1646        .iter()
1647        .any(|error_type| error_type.trim().is_empty())
1648    {
1649        return Err(ActivityOptionsError::new(
1650            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1651            Some("retry_policy.non_retryable_error_types"),
1652            "non_retryable_error_types must not contain empty values",
1653        ));
1654    }
1655
1656    let backoff_seconds = match &policy.backoff {
1657        None => None,
1658        Some(backoff) => {
1659            let max_attempts = policy.max_attempts.ok_or_else(|| {
1660                ActivityOptionsError::new(
1661                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1662                    Some("retry_policy.backoff"),
1663                    "backoff requires max_attempts",
1664                )
1665            })?;
1666            let retry_count = max_attempts.saturating_sub(1) as usize;
1667            let intervals = match backoff {
1668                ActivityBackoff::Explicit(intervals) => {
1669                    if intervals.len() > retry_count {
1670                        return Err(ActivityOptionsError::new(
1671                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
1672                            Some("retry_policy.backoff"),
1673                            "backoff interval count must not exceed max_attempts - 1",
1674                        ));
1675                    }
1676                    intervals.clone()
1677                }
1678                ActivityBackoff::Exponential {
1679                    initial_interval,
1680                    coefficient,
1681                    maximum_interval,
1682                } => {
1683                    if *coefficient < 1 {
1684                        return Err(ActivityOptionsError::new(
1685                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1686                            Some("retry_policy.backoff.coefficient"),
1687                            "backoff coefficient must be >= 1",
1688                        ));
1689                    }
1690                    if retry_count > 10_000 {
1691                        return Err(ActivityOptionsError::new(
1692                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1693                            Some("retry_policy.max_attempts"),
1694                            "generated backoff supports at most 10000 retry intervals",
1695                        ));
1696                    }
1697                    let mut current = *initial_interval;
1698                    let mut intervals = Vec::with_capacity(retry_count);
1699                    for _ in 0..retry_count {
1700                        let interval = maximum_interval
1701                            .map(|maximum| current.min(maximum))
1702                            .unwrap_or(current);
1703                        intervals.push(interval);
1704                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
1705                            break;
1706                        }
1707                        current = current.checked_mul(*coefficient).ok_or_else(|| {
1708                            ActivityOptionsError::new(
1709                                ActivityOptionsErrorKind::BackoffOverflow,
1710                                Some("retry_policy.backoff"),
1711                                "generated backoff interval overflowed",
1712                            )
1713                        })?;
1714                    }
1715                    intervals
1716                }
1717            };
1718            Some(
1719                intervals
1720                    .into_iter()
1721                    .map(|interval| {
1722                        activity_protocol_seconds(interval).ok_or_else(|| {
1723                            ActivityOptionsError::new(
1724                                ActivityOptionsErrorKind::BackoffOverflow,
1725                                Some("retry_policy.backoff"),
1726                                "backoff interval is too large for the worker protocol",
1727                            )
1728                        })
1729                    })
1730                    .collect::<std::result::Result<Vec<_>, _>>()?,
1731            )
1732        }
1733    };
1734
1735    let mut encoded = serde_json::Map::new();
1736    if let Some(max_attempts) = policy.max_attempts {
1737        encoded.insert("max_attempts".to_string(), json!(max_attempts));
1738    }
1739    if let Some(backoff_seconds) = backoff_seconds {
1740        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1741    }
1742    if !policy.non_retryable_error_types.is_empty() {
1743        let mut canonical_error_types = Vec::new();
1744        for error_type in policy
1745            .non_retryable_error_types
1746            .iter()
1747            .map(|error_type| error_type.trim())
1748        {
1749            if !canonical_error_types.contains(&error_type) {
1750                canonical_error_types.push(error_type);
1751            }
1752        }
1753        encoded.insert(
1754            "non_retryable_error_types".to_string(),
1755            json!(canonical_error_types),
1756        );
1757    }
1758    Ok(Value::Object(encoded))
1759}
1760
1761/// A stable, machine-readable failure raised when workflow code no longer
1762/// reconstructs the durable command stream recorded in history.
1763#[derive(Clone, Debug, Error)]
1764#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1765pub struct ReplayFailure {
1766    pub reason: String,
1767    pub sequence: Option<u64>,
1768    pub expected: Option<String>,
1769    pub actual: Option<String>,
1770    pub message: String,
1771}
1772
1773impl ReplayFailure {
1774    fn new(
1775        reason: impl Into<String>,
1776        sequence: Option<u64>,
1777        expected: Option<String>,
1778        actual: Option<String>,
1779        message: impl Into<String>,
1780    ) -> Self {
1781        Self {
1782            reason: reason.into(),
1783            sequence,
1784            expected,
1785            actual,
1786            message: message.into(),
1787        }
1788    }
1789}
1790
1791/// A stable, machine-readable workflow query or query-task settlement failure.
1792#[derive(Clone, Debug, Error)]
1793#[error("query failed ({reason}, HTTP {status}): {message}")]
1794pub struct QueryFailure {
1795    pub status: u16,
1796    pub reason: String,
1797    pub message: String,
1798    pub body: Value,
1799}
1800
1801/// A stable failure returned when a server rejects an SDK protocol version.
1802#[derive(Clone, Debug, Error)]
1803#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1804pub struct ProtocolFailure {
1805    pub status: u16,
1806    pub reason: String,
1807    pub message: String,
1808    pub supported_version: Option<String>,
1809    pub requested_version: Option<String>,
1810    pub body: Value,
1811}
1812
1813#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1814pub struct PayloadEnvelope {
1815    pub codec: String,
1816    pub blob: String,
1817}
1818
1819impl PayloadEnvelope {
1820    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1821        encode_payload(value, DEFAULT_CODEC)
1822    }
1823
1824    /// Encode an explicit typed value, including the bytes branch that JSON
1825    /// serialization cannot represent.
1826    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1827        encode_avro_value(value)
1828    }
1829}
1830
1831/// Native adapter for the fixed language-neutral Avro Value schema.
1832#[derive(Clone, Debug)]
1833pub enum AvroValue {
1834    Null,
1835    Boolean(bool),
1836    Long(i64),
1837    Double(f64),
1838    Bytes(Vec<u8>),
1839    String(String),
1840    Array(Vec<AvroValue>),
1841    Map(BTreeMap<String, AvroValue>),
1842}
1843
1844impl PartialEq for AvroValue {
1845    fn eq(&self, other: &Self) -> bool {
1846        match (self, other) {
1847            (Self::Null, Self::Null) => true,
1848            (Self::Boolean(left), Self::Boolean(right)) => left == right,
1849            (Self::Long(left), Self::Long(right)) => left == right,
1850            (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1851            (Self::Bytes(left), Self::Bytes(right)) => left == right,
1852            (Self::String(left), Self::String(right)) => left == right,
1853            (Self::Array(left), Self::Array(right)) => left == right,
1854            (Self::Map(left), Self::Map(right)) => left == right,
1855            _ => false,
1856        }
1857    }
1858}
1859
1860impl AvroValue {
1861    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1862        Self::from_serde_value(
1863            serde_value::to_value(value).map_err(|error| {
1864                Error::Codec(format!("could not adapt value for Avro: {error}"))
1865            })?,
1866        )
1867    }
1868
1869    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1870        use serde_value::Value as SerdeValue;
1871
1872        match value {
1873            SerdeValue::Unit => Ok(Self::Null),
1874            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1875            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1876            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1877            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1878            SerdeValue::I64(value) => Ok(Self::Long(value)),
1879            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1880            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1881            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1882            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1883                Error::Codec(
1884                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1885                        .to_string(),
1886                )
1887            }),
1888            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1889            SerdeValue::F64(value) => Self::finite_double(value),
1890            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1891            SerdeValue::String(value) => Ok(Self::String(value)),
1892            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1893            SerdeValue::Option(None) => Ok(Self::Null),
1894            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1895                Self::from_serde_value(*value)
1896            }
1897            SerdeValue::Seq(values) => values
1898                .into_iter()
1899                .map(Self::from_serde_value)
1900                .collect::<Result<Vec<_>>>()
1901                .map(Self::Array),
1902            SerdeValue::Map(values) => values
1903                .into_iter()
1904                .map(|(key, value)| {
1905                    let SerdeValue::String(key) = key else {
1906                        return Err(Error::Codec(
1907                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1908                        ));
1909                    };
1910
1911                    Ok((key, Self::from_serde_value(value)?))
1912                })
1913                .collect::<Result<BTreeMap<_, _>>>()
1914                .map(Self::Map),
1915        }
1916    }
1917
1918    fn finite_double(value: f64) -> Result<Self> {
1919        if !value.is_finite() {
1920            return Err(Error::Codec(
1921                "non_finite_float: Avro Value doubles must be finite".to_string(),
1922            ));
1923        }
1924
1925        Ok(Self::Double(value))
1926    }
1927
1928    fn into_json(self) -> Result<Value> {
1929        match self {
1930            Self::Null => Ok(Value::Null),
1931            Self::Boolean(value) => Ok(Value::Bool(value)),
1932            Self::Long(value) => Ok(Value::Number(value.into())),
1933            Self::Double(value) => serde_json::Number::from_f64(value)
1934                .map(Value::Number)
1935                .ok_or_else(|| {
1936                    Error::Codec(
1937                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1938                    )
1939                }),
1940            Self::Bytes(value) => Ok(json!({
1941                "$type": "bytes",
1942                "base64": BASE64.encode(value),
1943            })),
1944            Self::String(value) => Ok(Value::String(value)),
1945            Self::Array(values) => values
1946                .into_iter()
1947                .map(Self::into_json)
1948                .collect::<Result<Vec<_>>>()
1949                .map(Value::Array),
1950            Self::Map(values) => values
1951                .into_iter()
1952                .map(|(key, value)| Ok((key, value.into_json()?)))
1953                .collect::<Result<serde_json::Map<_, _>>>()
1954                .map(Value::Object),
1955        }
1956    }
1957
1958    fn into_serde_value(self) -> serde_value::Value {
1959        use serde_value::Value as SerdeValue;
1960
1961        match self {
1962            Self::Null => SerdeValue::Unit,
1963            Self::Boolean(value) => SerdeValue::Bool(value),
1964            Self::Long(value) => SerdeValue::I64(value),
1965            Self::Double(value) => SerdeValue::F64(value),
1966            Self::Bytes(value) => SerdeValue::Bytes(value),
1967            Self::String(value) => SerdeValue::String(value),
1968            Self::Array(values) => {
1969                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1970            }
1971            Self::Map(values) => SerdeValue::Map(
1972                values
1973                    .into_iter()
1974                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1975                    .collect(),
1976            ),
1977        }
1978    }
1979
1980    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1981        self.into_serde_value().deserialize_into().map_err(|error| {
1982            Error::Codec(format!(
1983                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1984            ))
1985        })
1986    }
1987}
1988
1989impl Serialize for AvroValue {
1990    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1991    where
1992        S: Serializer,
1993    {
1994        match self {
1995            Self::Null => serializer.serialize_unit(),
1996            Self::Boolean(value) => serializer.serialize_bool(*value),
1997            Self::Long(value) => serializer.serialize_i64(*value),
1998            Self::Double(value) => serializer.serialize_f64(*value),
1999            Self::Bytes(value) => serializer.serialize_bytes(value),
2000            Self::String(value) => serializer.serialize_str(value),
2001            Self::Array(values) => {
2002                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2003                for value in values {
2004                    sequence.serialize_element(value)?;
2005                }
2006                sequence.end()
2007            }
2008            Self::Map(values) => {
2009                let mut map = serializer.serialize_map(Some(values.len()))?;
2010                for (key, value) in values {
2011                    map.serialize_entry(key, value)?;
2012                }
2013                map.end()
2014            }
2015        }
2016    }
2017}
2018
2019pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2020    let datum = avro_value_to_datum(value)?;
2021    let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2022        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2023    let mut bytes = Vec::with_capacity(datum.len() + 10);
2024    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2025    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2026    bytes.extend_from_slice(&datum);
2027    Ok(PayloadEnvelope {
2028        codec: DEFAULT_CODEC.to_string(),
2029        blob: BASE64.encode(bytes),
2030    })
2031}
2032
2033pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2034    if envelope.codec != DEFAULT_CODEC {
2035        return Err(unsupported_payload_codec(&envelope.codec));
2036    }
2037    decode_avro_value_blob(&envelope.blob)
2038}
2039
2040pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2041    let blob = match codec {
2042        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2043        other => return Err(unsupported_payload_codec(other)),
2044    };
2045
2046    Ok(PayloadEnvelope {
2047        codec: codec.to_string(),
2048        blob,
2049    })
2050}
2051
2052pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2053    match envelope.codec.as_str() {
2054        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2055        other => Err(unsupported_payload_codec(other)),
2056    }
2057}
2058
2059fn handler_type_error<T>(
2060    handler_kind: HandlerKind,
2061    handler_name: &str,
2062    value_kind: HandlerValueKind,
2063    message: impl Into<String>,
2064) -> Error {
2065    Error::HandlerType {
2066        handler_kind,
2067        handler_name: handler_name.to_string(),
2068        value_kind,
2069        rust_type: type_name::<T>(),
2070        message: message.into(),
2071    }
2072}
2073
2074fn decode_handler_input<T: DeserializeOwned>(
2075    arguments: AvroValue,
2076    handler_kind: HandlerKind,
2077    handler_name: &str,
2078) -> Result<T> {
2079    let argument = match arguments {
2080        AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2081            arguments.pop().expect("one typed handler argument")
2082        }
2083        AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2084        AvroValue::Array(arguments) => {
2085            return Err(handler_type_error::<T>(
2086                handler_kind,
2087                handler_name,
2088                HandlerValueKind::Input,
2089                format!(
2090                    "typed handlers accept one request value, but the task carried {} arguments",
2091                    arguments.len()
2092                ),
2093            ));
2094        }
2095        argument => argument,
2096    };
2097
2098    argument.deserialize().map_err(|error| {
2099        handler_type_error::<T>(
2100            handler_kind,
2101            handler_name,
2102            HandlerValueKind::Input,
2103            error.to_string(),
2104        )
2105    })
2106}
2107
2108fn encode_handler_result<T: Serialize>(
2109    result: &T,
2110    handler_kind: HandlerKind,
2111    handler_name: &str,
2112) -> Result<AvroValue> {
2113    AvroValue::from_serialize(result).map_err(|error| {
2114        handler_type_error::<T>(
2115            handler_kind,
2116            handler_name,
2117            HandlerValueKind::Result,
2118            error.to_string(),
2119        )
2120    })
2121}
2122
2123fn decode_handler_result<T: DeserializeOwned>(
2124    result: AvroValue,
2125    handler_kind: HandlerKind,
2126    handler_name: &str,
2127) -> Result<T> {
2128    result.deserialize().map_err(|error| {
2129        handler_type_error::<T>(
2130            handler_kind,
2131            handler_name,
2132            HandlerValueKind::Result,
2133            error.to_string(),
2134        )
2135    })
2136}
2137
2138#[cfg(test)]
2139fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2140    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2141}
2142
2143fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2144    validate_payload_codec(fallback_codec)?;
2145
2146    if value.is_null() {
2147        return Ok(Value::Null);
2148    }
2149
2150    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2151        return decode_blob(blob, codec);
2152    }
2153
2154    if let Some(blob) = value.as_str() {
2155        return decode_blob(blob, fallback_codec);
2156    }
2157
2158    Err(untagged_payload_value())
2159}
2160
2161fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2162    let envelope = match codec {
2163        DEFAULT_CODEC => encode_avro_value(value)?,
2164        other => return Err(unsupported_payload_codec(other)),
2165    };
2166    Ok(serde_json::to_value(envelope)?)
2167}
2168
2169fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2170    validate_payload_codec(fallback_codec)?;
2171
2172    if value.is_null() {
2173        return Ok(AvroValue::Null);
2174    }
2175
2176    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2177        validate_payload_codec(codec)?;
2178        return decode_avro_value_blob(blob);
2179    }
2180
2181    if let Some(blob) = value.as_str() {
2182        return match fallback_codec {
2183            DEFAULT_CODEC => decode_avro_value_blob(blob),
2184            other => Err(unsupported_payload_codec(other)),
2185        };
2186    }
2187
2188    Err(untagged_payload_value())
2189}
2190
2191fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2192    match value {
2193        AvroValue::Null => AvroValue::Array(Vec::new()),
2194        AvroValue::Array(_) => value,
2195        other => AvroValue::Array(vec![other]),
2196    }
2197}
2198
2199fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2200    match codec {
2201        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2202        other => Err(unsupported_payload_codec(other)),
2203    }
2204}
2205
2206fn validate_payload_codec(codec: &str) -> Result<()> {
2207    match codec {
2208        DEFAULT_CODEC => Ok(()),
2209        MISSING_TASK_PAYLOAD_CODEC => {
2210            Err(invalid_task_payload_codec("task payload_codec is missing"))
2211        }
2212        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2213        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2214            "task payload_codec must be a string",
2215        )),
2216        other => Err(unsupported_payload_codec(other)),
2217    }
2218}
2219
2220fn invalid_task_payload_codec(reason: &str) -> Error {
2221    Error::Codec(format!(
2222        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2223    ))
2224}
2225
2226fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2227    let Some(object) = value.as_object() else {
2228        return Ok(None);
2229    };
2230    if !object.contains_key("codec") && !object.contains_key("blob") {
2231        return Ok(None);
2232    }
2233
2234    let codec = object
2235        .get("codec")
2236        .and_then(Value::as_str)
2237        .ok_or_else(invalid_payload_envelope)?;
2238    validate_payload_codec(codec)?;
2239    let blob = object
2240        .get("blob")
2241        .and_then(Value::as_str)
2242        .ok_or_else(invalid_payload_envelope)?;
2243    Ok(Some((codec, blob)))
2244}
2245
2246fn invalid_payload_envelope() -> Error {
2247    Error::Codec(
2248        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2249            .to_string(),
2250    )
2251}
2252
2253fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2254    for command in commands {
2255        let Some(command) = command.as_object() else {
2256            continue;
2257        };
2258        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2259            continue;
2260        };
2261        let Some(payload_field) = workflow_command_payload_field(command_type) else {
2262            continue;
2263        };
2264
2265        if let Some(codec) = command.get("payload_codec") {
2266            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2267            validate_payload_codec(codec)?;
2268        }
2269
2270        let payload = command
2271            .get(payload_field)
2272            .ok_or_else(invalid_payload_envelope)?;
2273        validate_outbound_payload_envelope(payload)?;
2274    }
2275    Ok(())
2276}
2277
2278fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2279    if commands.iter().any(|command| {
2280        command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2281            && command
2282                .get("condition_wait_occurrence_id")
2283                .and_then(Value::as_str)
2284                .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2285    }) {
2286        CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2287    } else if commands.iter().any(|command| {
2288        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2289            && command.get("attribute_types").is_some()
2290    }) {
2291        TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2292    } else if commands
2293        .iter()
2294        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2295    {
2296        MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2297    } else if commands
2298        .iter()
2299        .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2300    {
2301        CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2302    } else if commands.iter().any(|command| {
2303        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2304    }) {
2305        SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2306    } else {
2307        WORKER_PROTOCOL_VERSION
2308    }
2309}
2310
2311fn workflow_completion_protocol_version_with_message_streams(
2312    commands: &[Value],
2313    has_message_stream_metadata: bool,
2314) -> &'static str {
2315    let command_protocol = workflow_completion_protocol_version(commands);
2316    if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2317        MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2318    } else {
2319        command_protocol
2320    }
2321}
2322
2323fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2324    match command_type {
2325        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2326        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2327        "start_service_operation" => Some("request_payload"),
2328        "upsert_memo" => Some("entries"),
2329        _ => None,
2330    }
2331}
2332
2333fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2334    let Some((codec, blob)) = payload_envelope_parts(value)? else {
2335        return Err(untagged_payload_value());
2336    };
2337    validate_payload_codec(codec)?;
2338    decode_avro_value_blob(blob)?;
2339    Ok(())
2340}
2341
2342fn unsupported_payload_codec(codec: &str) -> Error {
2343    Error::Codec(format!(
2344        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2345    ))
2346}
2347
2348fn untagged_payload_value() -> Error {
2349    Error::Codec(
2350        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2351            .to_string(),
2352    )
2353}
2354
2355fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2356    let bytes = BASE64.decode(blob).map_err(|err| {
2357        Error::Codec(format!(
2358            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2359        ))
2360    })?;
2361
2362    if serde_json::from_slice::<Value>(&bytes).is_ok() {
2363        return Err(unsupported_payload_codec("json"));
2364    }
2365
2366    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2367        return Err(Error::Codec(
2368            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2369        ));
2370    }
2371
2372    let fingerprint: [u8; 8] = bytes[2..10]
2373        .try_into()
2374        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2375    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2376        return Err(Error::Codec(format!(
2377            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2378            fingerprint
2379                .iter()
2380                .map(|byte| format!("{byte:02x}"))
2381                .collect::<String>()
2382        )));
2383    }
2384
2385    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2386    // The current fingerprint selects the current immutable schema, so reader
2387    // resolution would only re-walk the same recursive union. Future retained
2388    // writer fingerprints supply a distinct reader schema in this branch.
2389    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2390    if datum_reader.truncated {
2391        return Err(Error::Codec(
2392            "invalid_payload_framing: truncated Avro Value datum".to_string(),
2393        ));
2394    }
2395    let datum = datum.map_err(|err| {
2396        Error::Codec(format!(
2397            "invalid_payload_framing: malformed Avro Value datum: {err}"
2398        ))
2399    })?;
2400    if datum_reader.remaining() != 0 {
2401        return Err(Error::Codec(format!(
2402            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2403            datum_reader.remaining()
2404        )));
2405    }
2406    avro_value_from_datum(datum)
2407}
2408
2409struct StrictAvroDatumReader<'a> {
2410    bytes: &'a [u8],
2411    offset: usize,
2412    truncated: bool,
2413}
2414
2415impl<'a> StrictAvroDatumReader<'a> {
2416    fn new(bytes: &'a [u8]) -> Self {
2417        Self {
2418            bytes,
2419            offset: 0,
2420            truncated: false,
2421        }
2422    }
2423
2424    fn remaining(&self) -> usize {
2425        self.bytes.len() - self.offset
2426    }
2427}
2428
2429impl Read for StrictAvroDatumReader<'_> {
2430    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2431        let count = buffer.len().min(self.remaining());
2432        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2433        self.offset += count;
2434        if count < buffer.len() {
2435            self.truncated = true;
2436        }
2437
2438        Ok(count)
2439    }
2440}
2441
2442fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2443    let branch = match value {
2444        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2445        AvroValue::Boolean(value) => AvroDatum::Union(
2446            1,
2447            Box::new(AvroDatum::Record(vec![(
2448                "boolean".to_string(),
2449                AvroDatum::Boolean(*value),
2450            )])),
2451        ),
2452        AvroValue::Long(value) => AvroDatum::Union(
2453            2,
2454            Box::new(AvroDatum::Record(vec![(
2455                "long".to_string(),
2456                AvroDatum::Long(*value),
2457            )])),
2458        ),
2459        AvroValue::Double(value) => {
2460            if !value.is_finite() {
2461                return Err(Error::Codec(
2462                    "non_finite_float: Avro Value doubles must be finite".to_string(),
2463                ));
2464            }
2465            AvroDatum::Union(
2466                3,
2467                Box::new(AvroDatum::Record(vec![(
2468                    "double".to_string(),
2469                    AvroDatum::Double(*value),
2470                )])),
2471            )
2472        }
2473        AvroValue::Bytes(value) => AvroDatum::Union(
2474            4,
2475            Box::new(AvroDatum::Record(vec![(
2476                "bytes".to_string(),
2477                AvroDatum::Bytes(value.clone()),
2478            )])),
2479        ),
2480        AvroValue::String(value) => AvroDatum::Union(
2481            5,
2482            Box::new(AvroDatum::Record(vec![(
2483                "string".to_string(),
2484                AvroDatum::String(value.clone()),
2485            )])),
2486        ),
2487        AvroValue::Array(values) => AvroDatum::Union(
2488            6,
2489            Box::new(AvroDatum::Record(vec![(
2490                "items".to_string(),
2491                AvroDatum::Array(
2492                    values
2493                        .iter()
2494                        .map(avro_value_to_datum)
2495                        .collect::<Result<Vec<_>>>()?,
2496                ),
2497            )])),
2498        ),
2499        AvroValue::Map(values) => AvroDatum::Union(
2500            7,
2501            Box::new(AvroDatum::Record(vec![(
2502                "entries".to_string(),
2503                AvroDatum::Array(
2504                    values
2505                        .iter()
2506                        .map(|(key, value)| {
2507                            Ok(AvroDatum::Record(vec![
2508                                ("key".to_string(), AvroDatum::String(key.clone())),
2509                                ("value".to_string(), avro_value_to_datum(value)?),
2510                            ]))
2511                        })
2512                        .collect::<Result<Vec<_>>>()?,
2513                ),
2514            )])),
2515        ),
2516    };
2517    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2518}
2519
2520fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2521    let AvroDatum::Record(mut outer) = datum else {
2522        return Err(Error::Codec(
2523            "invalid_payload_framing: datum is not a Value record".to_string(),
2524        ));
2525    };
2526    let (_, branch) = outer
2527        .pop()
2528        .filter(|(name, _)| name == "value")
2529        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2530    let AvroDatum::Union(_, branch) = branch else {
2531        return Err(Error::Codec(
2532            "invalid_payload_framing: invalid Value union".to_string(),
2533        ));
2534    };
2535    match *branch {
2536        AvroDatum::Null => Ok(AvroValue::Null),
2537        AvroDatum::Record(mut fields) => {
2538            let (name, value) = fields.pop().ok_or_else(|| {
2539                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2540            })?;
2541            match (name.as_str(), value) {
2542                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2543                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2544                ("double", AvroDatum::Double(value)) if value.is_finite() => {
2545                    Ok(AvroValue::Double(value))
2546                }
2547                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2548                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2549                ("items", AvroDatum::Array(values)) => values
2550                    .into_iter()
2551                    .map(avro_value_from_datum)
2552                    .collect::<Result<Vec<_>>>()
2553                    .map(AvroValue::Array),
2554                ("entries", AvroDatum::Map(values)) => values
2555                    .into_iter()
2556                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2557                    .collect::<Result<BTreeMap<_, _>>>()
2558                    .map(AvroValue::Map),
2559                _ => Err(Error::Codec(
2560                    "invalid_payload_framing: unknown Value branch".to_string(),
2561                )),
2562            }
2563        }
2564        _ => Err(Error::Codec(
2565            "invalid_payload_framing: invalid Value branch".to_string(),
2566        )),
2567    }
2568}
2569
2570fn avro_value_schema() -> Result<&'static Schema> {
2571    match AVRO_VALUE_SCHEMA.get_or_init(|| {
2572        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2573            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2574    }) {
2575        Ok(schema) => Ok(schema),
2576        Err(message) => Err(Error::Codec(message.clone())),
2577    }
2578}
2579
2580fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2581    match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2582        // Apache Avro's Value::Map uses a randomized HashMap. Arrays and maps
2583        // have the same block representation when each ordered array record
2584        // contains the map key followed by its value, so this encoding-only
2585        // adaptation lets the official encoder retain BTreeMap wire order.
2586        let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2587            .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2588        let entries_schema = schema
2589            .pointer_mut("/fields/0/type/7/fields/0/type")
2590            .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2591        if *entries_schema != json!({"type": "map", "values": "Value"}) {
2592            return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2593        }
2594        *entries_schema = json!({
2595            "type": "array",
2596            "items": {
2597                "type": "record",
2598                "name": "MapEntry",
2599                "fields": [
2600                    {"name": "key", "type": "string"},
2601                    {"name": "value", "type": "Value"}
2602                ]
2603            }
2604        });
2605        Schema::parse_str(&schema.to_string())
2606            .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2607    }) {
2608        Ok(schema) => Ok(schema),
2609        Err(message) => Err(Error::Codec(message.clone())),
2610    }
2611}
2612
2613#[derive(Clone, Debug)]
2614pub struct Client {
2615    http: reqwest::Client,
2616    base_url: String,
2617    token: Option<String>,
2618    control_token: Option<String>,
2619    worker_token: Option<String>,
2620    namespace: String,
2621    worker_storage_admission: Option<WorkerStorageAdmission>,
2622}
2623
2624impl Client {
2625    pub fn new(base_url: impl Into<String>) -> Result<Self> {
2626        Self::builder(base_url).build()
2627    }
2628
2629    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2630        ClientBuilder {
2631            base_url: base_url.into(),
2632            token: None,
2633            control_token: None,
2634            worker_token: None,
2635            namespace: "default".to_string(),
2636            timeout: Duration::from_secs(60),
2637        }
2638    }
2639
2640    pub async fn health(&self) -> Result<Value> {
2641        self.request_json(
2642            reqwest::Method::GET,
2643            "/health",
2644            RequestProtocol::ControlPlane,
2645            Option::<&Value>::None,
2646        )
2647        .await
2648    }
2649
2650    pub async fn cluster_info(&self) -> Result<Value> {
2651        self.request_json(
2652            reqwest::Method::GET,
2653            "/cluster/info",
2654            RequestProtocol::ControlPlane,
2655            Option::<&Value>::None,
2656        )
2657        .await
2658    }
2659
2660    pub async fn start_workflow<T: Serialize>(
2661        &self,
2662        workflow_type: &str,
2663        task_queue: &str,
2664        workflow_id: &str,
2665        input: T,
2666    ) -> Result<WorkflowHandle> {
2667        self.start_workflow_with_options(
2668            workflow_type,
2669            task_queue,
2670            workflow_id,
2671            WorkflowStartOptions::default(),
2672            input,
2673        )
2674        .await
2675    }
2676
2677    /// Start a workflow with explicit server-enforced execution and run
2678    /// deadlines.
2679    pub async fn start_workflow_with_options<T: Serialize>(
2680        &self,
2681        workflow_type: &str,
2682        task_queue: &str,
2683        workflow_id: &str,
2684        options: WorkflowStartOptions,
2685        input: T,
2686    ) -> Result<WorkflowHandle> {
2687        options.validate()?;
2688        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2689        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2690        let body = json!({
2691            "workflow_id": workflow_id,
2692            "workflow_type": workflow_type,
2693            "task_queue": task_queue,
2694            "input": input_envelope,
2695            "execution_timeout_seconds": options.execution_timeout_seconds,
2696            "run_timeout_seconds": options.run_timeout_seconds
2697        });
2698
2699        let data: Value = self
2700            .request_json(
2701                reqwest::Method::POST,
2702                "/workflows",
2703                RequestProtocol::ControlPlane,
2704                Some(&body),
2705            )
2706            .await?;
2707
2708        Ok(WorkflowHandle {
2709            client: self.clone(),
2710            workflow_id: data
2711                .get("workflow_id")
2712                .and_then(Value::as_str)
2713                .unwrap_or(workflow_id)
2714                .to_string(),
2715            run_id: data
2716                .get("run_id")
2717                .and_then(Value::as_str)
2718                .map(str::to_string),
2719            workflow_type: data
2720                .get("workflow_type")
2721                .and_then(Value::as_str)
2722                .unwrap_or(workflow_type)
2723                .to_string(),
2724        })
2725    }
2726
2727    pub async fn signal_workflow<T: Serialize>(
2728        &self,
2729        workflow_id: &str,
2730        signal_name: &str,
2731        input: T,
2732    ) -> Result<Value> {
2733        self.signal_workflow_target(workflow_id, None, signal_name, input)
2734            .await
2735    }
2736
2737    /// Append one idempotently identified item to an instance-scoped input stream.
2738    pub async fn append_message_stream<T: Serialize>(
2739        &self,
2740        workflow_id: &str,
2741        stream_name: &str,
2742        message_id: &str,
2743        input: T,
2744    ) -> Result<Value> {
2745        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2746        let body = json!({
2747            "message_id": message_id,
2748            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2749        });
2750        self.request_json(
2751            reqwest::Method::POST,
2752            &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2753            RequestProtocol::ControlPlane,
2754            Some(&body),
2755        )
2756        .await
2757    }
2758
2759    /// Signal only if `run_id` is still the current run for this instance.
2760    pub async fn signal_workflow_run<T: Serialize>(
2761        &self,
2762        workflow_id: &str,
2763        run_id: &str,
2764        signal_name: &str,
2765        input: T,
2766    ) -> Result<Value> {
2767        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2768            .await
2769    }
2770
2771    async fn signal_workflow_target<T: Serialize>(
2772        &self,
2773        workflow_id: &str,
2774        run_id: Option<&str>,
2775        signal_name: &str,
2776        input: T,
2777    ) -> Result<Value> {
2778        validate_user_signal_name(signal_name)?;
2779        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2780        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2781        let body = json!({
2782            "input": input_envelope
2783        });
2784        let path = match run_id {
2785            Some(run_id) => {
2786                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2787            }
2788            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2789        };
2790        self.request_json(
2791            reqwest::Method::POST,
2792            &path,
2793            RequestProtocol::ControlPlane,
2794            Some(&body),
2795        )
2796        .await
2797    }
2798
2799    /// Request cooperative cancellation of the current run for an instance.
2800    pub async fn cancel_workflow(
2801        &self,
2802        workflow_id: &str,
2803        options: WorkflowCommandOptions,
2804    ) -> Result<WorkflowCommandResult> {
2805        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2806            .await
2807    }
2808
2809    /// Request cooperative cancellation only if `run_id` is still current.
2810    pub async fn cancel_workflow_run(
2811        &self,
2812        workflow_id: &str,
2813        run_id: &str,
2814        options: WorkflowCommandOptions,
2815    ) -> Result<WorkflowCommandResult> {
2816        self.workflow_command(
2817            workflow_id,
2818            Some(run_id),
2819            WorkflowCommandKind::Cancel,
2820            options,
2821        )
2822        .await
2823    }
2824
2825    /// Forcefully terminate the current run for an instance.
2826    pub async fn terminate_workflow(
2827        &self,
2828        workflow_id: &str,
2829        options: WorkflowCommandOptions,
2830    ) -> Result<WorkflowCommandResult> {
2831        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2832            .await
2833    }
2834
2835    /// Forcefully terminate only if `run_id` is still current.
2836    pub async fn terminate_workflow_run(
2837        &self,
2838        workflow_id: &str,
2839        run_id: &str,
2840        options: WorkflowCommandOptions,
2841    ) -> Result<WorkflowCommandResult> {
2842        self.workflow_command(
2843            workflow_id,
2844            Some(run_id),
2845            WorkflowCommandKind::Terminate,
2846            options,
2847        )
2848        .await
2849    }
2850
2851    async fn workflow_command(
2852        &self,
2853        workflow_id: &str,
2854        run_id: Option<&str>,
2855        command: WorkflowCommandKind,
2856        options: WorkflowCommandOptions,
2857    ) -> Result<WorkflowCommandResult> {
2858        let path = match run_id {
2859            Some(run_id) => format!(
2860                "/workflows/{workflow_id}/runs/{run_id}/{}",
2861                command.as_str()
2862            ),
2863            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2864        };
2865        let data = match self
2866            .request_json(
2867                reqwest::Method::POST,
2868                &path,
2869                RequestProtocol::ControlPlane,
2870                Some(&options),
2871            )
2872            .await
2873        {
2874            Ok(data) => data,
2875            Err(Error::Http { status, body }) => {
2876                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2877                    command,
2878                    status,
2879                    body,
2880                    workflow_id,
2881                    run_id,
2882                )));
2883            }
2884            Err(error) => return Err(error),
2885        };
2886
2887        Ok(workflow_command_result(command, data, workflow_id, run_id))
2888    }
2889
2890    /// Execute a named, read-only query against a running or completed workflow.
2891    ///
2892    /// Arguments and results use the platform payload envelope. Server and
2893    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
2894    /// reason, HTTP status, and original response body.
2895    pub async fn query_workflow<T: Serialize>(
2896        &self,
2897        workflow_id: &str,
2898        query_name: &str,
2899        input: T,
2900    ) -> Result<Value> {
2901        self.query_workflow_target(workflow_id, None, query_name, input)
2902            .await
2903    }
2904
2905    /// Query only if `run_id` is still current, preventing accidental retargeting.
2906    pub async fn query_workflow_run<T: Serialize>(
2907        &self,
2908        workflow_id: &str,
2909        run_id: &str,
2910        query_name: &str,
2911        input: T,
2912    ) -> Result<Value> {
2913        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2914            .await
2915    }
2916
2917    /// Query a workflow and return the lossless fixed Avro Value result.
2918    pub async fn query_workflow_avro_value<T: Serialize>(
2919        &self,
2920        workflow_id: &str,
2921        query_name: &str,
2922        input: T,
2923    ) -> Result<AvroValue> {
2924        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2925            .await
2926    }
2927
2928    /// Query a selected run and return the lossless fixed Avro Value result.
2929    pub async fn query_workflow_run_avro_value<T: Serialize>(
2930        &self,
2931        workflow_id: &str,
2932        run_id: &str,
2933        query_name: &str,
2934        input: T,
2935    ) -> Result<AvroValue> {
2936        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2937            .await
2938    }
2939
2940    async fn query_workflow_avro_value_target<T: Serialize>(
2941        &self,
2942        workflow_id: &str,
2943        run_id: Option<&str>,
2944        query_name: &str,
2945        input: T,
2946    ) -> Result<AvroValue> {
2947        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2948        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2949        let path = match run_id {
2950            Some(run_id) => {
2951                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2952            }
2953            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2954        };
2955        let response: Value = match self
2956            .request_json(
2957                reqwest::Method::POST,
2958                &path,
2959                RequestProtocol::ControlPlane,
2960                Some(&body),
2961            )
2962            .await
2963        {
2964            Ok(response) => response,
2965            Err(Error::Http { status, body }) => {
2966                return Err(Error::QueryFailed(query_failure(status, body)));
2967            }
2968            Err(error) => return Err(error),
2969        };
2970
2971        let envelope = response
2972            .get("result_envelope")
2973            .filter(|envelope| !envelope.is_null())
2974            .ok_or_else(|| {
2975                Error::Codec(
2976                    "missing_payload_envelope: typed query result requires result_envelope"
2977                        .to_string(),
2978                )
2979            })?;
2980        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2981    }
2982
2983    async fn query_workflow_target<T: Serialize>(
2984        &self,
2985        workflow_id: &str,
2986        run_id: Option<&str>,
2987        query_name: &str,
2988        input: T,
2989    ) -> Result<Value> {
2990        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2991        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2992        let body = json!({
2993            "input": input_envelope
2994        });
2995        let path = match run_id {
2996            Some(run_id) => {
2997                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2998            }
2999            None => format!("/workflows/{workflow_id}/query/{query_name}"),
3000        };
3001        let response: Value = match self
3002            .request_json(
3003                reqwest::Method::POST,
3004                &path,
3005                RequestProtocol::ControlPlane,
3006                Some(&body),
3007            )
3008            .await
3009        {
3010            Ok(response) => response,
3011            Err(Error::Http { status, body }) => {
3012                return Err(Error::QueryFailed(query_failure(status, body)));
3013            }
3014            Err(error) => return Err(error),
3015        };
3016
3017        if let Some(envelope) = response
3018            .get("result_envelope")
3019            .filter(|envelope| !envelope.is_null())
3020        {
3021            return decode_wire_value(envelope, DEFAULT_CODEC);
3022        }
3023
3024        Ok(response.get("result").cloned().unwrap_or(Value::Null))
3025    }
3026
3027    /// Send a synchronous update using fixed Avro Value arguments.
3028    pub async fn update_workflow<T: Serialize>(
3029        &self,
3030        workflow_id: &str,
3031        update_name: &str,
3032        input: T,
3033        request_id: Option<&str>,
3034    ) -> Result<Value> {
3035        let response = self
3036            .update_workflow_response(workflow_id, update_name, input, request_id)
3037            .await?;
3038        if let Some(envelope) = response
3039            .get("result_envelope")
3040            .filter(|envelope| !envelope.is_null())
3041        {
3042            return decode_wire_value(envelope, DEFAULT_CODEC);
3043        }
3044        Ok(response.get("result").cloned().unwrap_or(response))
3045    }
3046
3047    /// Send a synchronous update and retain a bytes-capable Avro result.
3048    pub async fn update_workflow_avro_value<T: Serialize>(
3049        &self,
3050        workflow_id: &str,
3051        update_name: &str,
3052        input: T,
3053        request_id: Option<&str>,
3054    ) -> Result<AvroValue> {
3055        let response = self
3056            .update_workflow_response(workflow_id, update_name, input, request_id)
3057            .await?;
3058        let envelope = response
3059            .get("result_envelope")
3060            .filter(|envelope| !envelope.is_null())
3061            .ok_or_else(|| {
3062                Error::Codec(
3063                    "missing_payload_envelope: typed update result requires result_envelope"
3064                        .to_string(),
3065                )
3066            })?;
3067        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3068    }
3069
3070    async fn update_workflow_response<T: Serialize>(
3071        &self,
3072        workflow_id: &str,
3073        update_name: &str,
3074        input: T,
3075        request_id: Option<&str>,
3076    ) -> Result<Value> {
3077        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3078        let mut body = json!({
3079            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3080            "wait_for": "completed",
3081        });
3082        if let Some(request_id) = request_id {
3083            body["request_id"] = json!(request_id);
3084        }
3085        self.request_json(
3086            reqwest::Method::POST,
3087            &format!("/workflows/{workflow_id}/update/{update_name}"),
3088            RequestProtocol::ControlPlane,
3089            Some(&body),
3090        )
3091        .await
3092    }
3093
3094    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3095        let path = format!("/workflows/{workflow_id}");
3096        let mut data: WorkflowDescription = self
3097            .request_json(
3098                reqwest::Method::GET,
3099                &path,
3100                RequestProtocol::ControlPlane,
3101                Option::<&Value>::None,
3102            )
3103            .await?;
3104        data.decode_payloads()?;
3105        Ok(data)
3106    }
3107
3108    /// Describe one selected run, including historical terminal runs.
3109    pub async fn describe_workflow_run(
3110        &self,
3111        workflow_id: &str,
3112        run_id: &str,
3113    ) -> Result<WorkflowDescription> {
3114        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3115        let mut data: WorkflowDescription = self
3116            .request_json(
3117                reqwest::Method::GET,
3118                &path,
3119                RequestProtocol::ControlPlane,
3120                Option::<&Value>::None,
3121            )
3122            .await?;
3123        data.decode_payloads()?;
3124        Ok(data)
3125    }
3126
3127    fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3128        let mut path = format!(
3129            "/workflows/{}/runs/{}/streams",
3130            percent_encode_path_segment(workflow_id),
3131            percent_encode_path_segment(run_id),
3132        );
3133        if let Some(stream_name) = stream_name {
3134            path.push('/');
3135            path.push_str(&percent_encode_path_segment(stream_name));
3136        }
3137        path
3138    }
3139
3140    /// List the run-scoped output streams already opened by a workflow.
3141    pub async fn list_workflow_streams(
3142        &self,
3143        workflow_id: &str,
3144        run_id: &str,
3145    ) -> Result<Vec<WorkflowStreamDescription>> {
3146        let response: WorkflowStreamListResponse = self
3147            .request_json(
3148                reqwest::Method::GET,
3149                &Self::workflow_stream_path(workflow_id, run_id, None),
3150                RequestProtocol::ControlPlane,
3151                Option::<&Value>::None,
3152            )
3153            .await?;
3154        Ok(response.streams)
3155    }
3156
3157    /// Describe stream lifecycle, offsets, pending count, and terminal error.
3158    pub async fn describe_workflow_stream(
3159        &self,
3160        workflow_id: &str,
3161        run_id: &str,
3162        stream_name: &str,
3163    ) -> Result<WorkflowStreamDescription> {
3164        let response: WorkflowStreamDescriptionResponse = self
3165            .request_json(
3166                reqwest::Method::GET,
3167                &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3168                RequestProtocol::ControlPlane,
3169                Option::<&Value>::None,
3170            )
3171            .await?;
3172        Ok(response.stream)
3173    }
3174
3175    /// Read one bounded page beginning at a zero-based offset.
3176    ///
3177    /// Delivery is at least once: persist `next_offset` only after processing
3178    /// the page. The future is cancellation-safe; dropping it cancels the
3179    /// in-flight request. Long polling is capped at 60 seconds by the SDK and
3180    /// service contract.
3181    pub async fn subscribe_workflow_stream(
3182        &self,
3183        workflow_id: &str,
3184        run_id: &str,
3185        stream_name: &str,
3186        from_offset: u64,
3187        max_items: usize,
3188        wait: Duration,
3189    ) -> Result<WorkflowStreamPage> {
3190        let max_items = max_items.clamp(1, 500);
3191        let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3192        let path = format!(
3193            "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3194            Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3195        );
3196        let response: WorkflowStreamPageResponse = self
3197            .request_json_with_timeout(
3198                reqwest::Method::GET,
3199                &path,
3200                RequestProtocol::ControlPlane,
3201                Option::<&Value>::None,
3202                Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3203            )
3204            .await?;
3205
3206        let items = response
3207            .items
3208            .into_iter()
3209            .map(|raw| {
3210                let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3211                let envelope = raw.get("payload").cloned();
3212                let payload = envelope
3213                    .as_ref()
3214                    .filter(|value| value.get("blob").is_some())
3215                    .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3216                    .transpose()?
3217                    .map(AvroValue::into_json)
3218                    .transpose()?;
3219                Ok(WorkflowStreamItem {
3220                    offset,
3221                    payload,
3222                    payload_envelope: envelope,
3223                    payload_reference: raw
3224                        .get("payload_reference")
3225                        .and_then(Value::as_str)
3226                        .map(str::to_string),
3227                    payload_codec: raw
3228                        .get("payload_codec")
3229                        .and_then(Value::as_str)
3230                        .map(str::to_string),
3231                    idempotency_key: raw
3232                        .get("idempotency_key")
3233                        .and_then(Value::as_str)
3234                        .map(str::to_string),
3235                    item_type: raw
3236                        .get("item_type")
3237                        .and_then(Value::as_str)
3238                        .map(str::to_string),
3239                    content_type: raw
3240                        .get("content_type")
3241                        .and_then(Value::as_str)
3242                        .map(str::to_string),
3243                    origin: raw
3244                        .get("origin")
3245                        .and_then(Value::as_str)
3246                        .map(str::to_string),
3247                    origin_reference: raw
3248                        .get("origin_reference")
3249                        .and_then(Value::as_str)
3250                        .map(str::to_string),
3251                    emitted_at: raw
3252                        .get("emitted_at")
3253                        .and_then(Value::as_str)
3254                        .map(str::to_string),
3255                    raw,
3256                })
3257            })
3258            .collect::<Result<Vec<_>>>()?;
3259        Ok(WorkflowStreamPage {
3260            stream: response.stream,
3261            items,
3262            next_offset: response.next_offset,
3263            terminal: response.terminal,
3264        })
3265    }
3266
3267    /// Append inline Avro envelopes or opaque external payload references.
3268    pub async fn append_workflow_stream(
3269        &self,
3270        workflow_id: &str,
3271        run_id: &str,
3272        stream_name: &str,
3273        items: &[WorkflowStreamAppendItem],
3274        max_pending_items: Option<u64>,
3275    ) -> Result<WorkflowStreamAppendResult> {
3276        if items.is_empty() {
3277            return Err(Error::Codec(
3278                "workflow_stream_items_empty: append requires at least one item".to_string(),
3279            ));
3280        }
3281        let mut body = json!({
3282            "items": items
3283                .iter()
3284                .map(|item| item.wire_value(None))
3285                .collect::<Vec<_>>(),
3286        });
3287        if let Some(max_pending_items) = max_pending_items {
3288            if max_pending_items == 0 {
3289                return Err(Error::Codec(
3290                    "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3291                        .to_string(),
3292                ));
3293            }
3294            body["max_pending_items"] = json!(max_pending_items);
3295        }
3296        let response: WorkflowStreamAppendResponse = self
3297            .request_json(
3298                reqwest::Method::POST,
3299                &format!(
3300                    "{}/items",
3301                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3302                ),
3303                RequestProtocol::ControlPlane,
3304                Some(&body),
3305            )
3306            .await?;
3307        Ok(WorkflowStreamAppendResult {
3308            stream: response.stream,
3309            accepted_offsets: response.accepted_offsets,
3310            accepted: response.accepted,
3311            deduped: response.deduped,
3312        })
3313    }
3314
3315    /// Close a stream, or mark it errored when `error_reason` is supplied.
3316    pub async fn close_workflow_stream(
3317        &self,
3318        workflow_id: &str,
3319        run_id: &str,
3320        stream_name: &str,
3321        error_reason: Option<&str>,
3322        retention_seconds: Option<u64>,
3323    ) -> Result<WorkflowStreamDescription> {
3324        let mut body = json!({});
3325        if let Some(error_reason) = error_reason {
3326            body["error_reason"] = json!(error_reason);
3327        }
3328        if let Some(retention_seconds) = retention_seconds {
3329            if retention_seconds == 0 {
3330                return Err(Error::Codec(
3331                    "workflow_stream_retention_invalid: retention_seconds must be positive"
3332                        .to_string(),
3333                ));
3334            }
3335            body["retention_seconds"] = json!(retention_seconds);
3336        }
3337        let response: WorkflowStreamDescriptionResponse = self
3338            .request_json(
3339                reqwest::Method::POST,
3340                &format!(
3341                    "{}/close",
3342                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3343                ),
3344                RequestProtocol::ControlPlane,
3345                Some(&body),
3346            )
3347            .await?;
3348        Ok(response.stream)
3349    }
3350
3351    pub async fn register_worker(
3352        &self,
3353        worker_id: &str,
3354        task_queue: &str,
3355        supported_workflow_types: Vec<String>,
3356        supported_activity_types: Vec<String>,
3357        max_concurrent_workflow_tasks: usize,
3358        max_concurrent_activity_tasks: usize,
3359    ) -> Result<RegisterWorkerResponse> {
3360        self.register_worker_with_capabilities(
3361            worker_id,
3362            task_queue,
3363            supported_workflow_types,
3364            supported_activity_types,
3365            max_concurrent_workflow_tasks,
3366            max_concurrent_activity_tasks,
3367            Vec::new(),
3368        )
3369        .await
3370    }
3371
3372    /// Register a worker and explicitly advertise additive worker capabilities.
3373    pub async fn register_worker_with_capabilities(
3374        &self,
3375        worker_id: &str,
3376        task_queue: &str,
3377        supported_workflow_types: Vec<String>,
3378        supported_activity_types: Vec<String>,
3379        max_concurrent_workflow_tasks: usize,
3380        max_concurrent_activity_tasks: usize,
3381        capabilities: Vec<String>,
3382    ) -> Result<RegisterWorkerResponse> {
3383        self.register_worker_with_command_contracts(
3384            worker_id,
3385            task_queue,
3386            supported_workflow_types,
3387            supported_activity_types,
3388            max_concurrent_workflow_tasks,
3389            max_concurrent_activity_tasks,
3390            capabilities,
3391            Value::Object(serde_json::Map::new()),
3392        )
3393        .await
3394    }
3395
3396    /// Register a worker and advertise its named query and update handlers.
3397    ///
3398    /// This Rust SDK cannot execute synchronous pre-accept update validation,
3399    /// so a workflow contract with a non-empty or malformed
3400    /// `update_validators` declaration returns
3401    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
3402    #[allow(clippy::too_many_arguments)]
3403    pub async fn register_worker_with_command_contracts(
3404        &self,
3405        worker_id: &str,
3406        task_queue: &str,
3407        supported_workflow_types: Vec<String>,
3408        supported_activity_types: Vec<String>,
3409        max_concurrent_workflow_tasks: usize,
3410        max_concurrent_activity_tasks: usize,
3411        capabilities: Vec<String>,
3412        workflow_command_contracts: Value,
3413    ) -> Result<RegisterWorkerResponse> {
3414        if let Some(contracts) = workflow_command_contracts.as_object() {
3415            for (workflow_type, contract) in contracts {
3416                let Some(update_validators) = contract.get("update_validators") else {
3417                    continue;
3418                };
3419                if !update_validators
3420                    .as_array()
3421                    .is_some_and(|validators| validators.is_empty())
3422                {
3423                    return Err(Error::UnsupportedUpdateValidators {
3424                        workflow_type: workflow_type.clone(),
3425                    });
3426                }
3427            }
3428        }
3429
3430        let mut body = json!({
3431            "worker_id": worker_id,
3432            "task_queue": task_queue,
3433            "runtime": "rust",
3434            "sdk_version": SDK_VERSION,
3435            "supported_workflow_types": supported_workflow_types,
3436            "supported_activity_types": supported_activity_types,
3437            "capabilities": capabilities,
3438            "capability_manifest": portable_worker_affinity_capability_manifest(),
3439            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3440            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3441        });
3442        if workflow_command_contracts
3443            .as_object()
3444            .is_some_and(|contracts| !contracts.is_empty())
3445        {
3446            body["workflow_command_contracts"] = workflow_command_contracts;
3447        }
3448
3449        self.request_json(
3450            reqwest::Method::POST,
3451            "/worker/register",
3452            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3453            Some(&body),
3454        )
3455        .await
3456    }
3457
3458    /// Gracefully remove one worker's registration through the worker plane.
3459    ///
3460    /// This operation is separate from operator-facing worker management. It
3461    /// uses worker-protocol authentication and returns the server's lease
3462    /// recovery result.
3463    pub async fn deregister_worker_registration(
3464        &self,
3465        worker_id: &str,
3466    ) -> Result<WorkerDeregistrationEnvelope> {
3467        let path = format!(
3468            "/worker/registrations/{}",
3469            percent_encode_path_segment(worker_id)
3470        );
3471        self.request_json(
3472            reqwest::Method::DELETE,
3473            &path,
3474            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3475            Option::<&Value>::None,
3476        )
3477        .await
3478    }
3479
3480    /// Long-poll for an ephemeral, read-only workflow query task.
3481    pub async fn poll_query_task(
3482        &self,
3483        worker_id: &str,
3484        task_queue: &str,
3485        timeout: Duration,
3486    ) -> Result<Option<QueryTask>> {
3487        Ok(self
3488            .poll_query_task_response(worker_id, task_queue, timeout)
3489            .await?
3490            .task)
3491    }
3492
3493    /// Poll a query task while preserving server stop and drain metadata.
3494    pub async fn poll_query_task_response(
3495        &self,
3496        worker_id: &str,
3497        task_queue: &str,
3498        timeout: Duration,
3499    ) -> Result<PollQueryTaskResponse> {
3500        let poll_request_id = unique_request_id("rust-query-poll");
3501        self.poll_query_task_response_with_request_id(
3502            worker_id,
3503            task_queue,
3504            timeout,
3505            &poll_request_id,
3506            1,
3507        )
3508        .await
3509    }
3510
3511    async fn poll_query_task_response_with_request_id(
3512        &self,
3513        worker_id: &str,
3514        task_queue: &str,
3515        timeout: Duration,
3516        poll_request_id: &str,
3517        transport_retries: usize,
3518    ) -> Result<PollQueryTaskResponse> {
3519        let timeout_seconds = long_poll_timeout_seconds(timeout);
3520        let body = json!({
3521            "worker_id": worker_id,
3522            "task_queue": task_queue,
3523            "poll_request_id": poll_request_id,
3524            "timeout_seconds": timeout_seconds,
3525        });
3526        self.poll_request_json(
3527            "/worker/query-tasks/poll",
3528            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3529            &body,
3530            timeout + Duration::from_secs(5),
3531            transport_retries,
3532        )
3533        .await
3534    }
3535
3536    /// Complete a query task without appending workflow history.
3537    pub async fn complete_query_task<T: Serialize>(
3538        &self,
3539        query_task_id: &str,
3540        lease_owner: &str,
3541        query_task_attempt: u64,
3542        result: T,
3543        codec: &str,
3544    ) -> Result<Value> {
3545        let typed_result = AvroValue::from_serialize(&result)?;
3546        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3547        self.complete_query_task_with_envelope(
3548            query_task_id,
3549            lease_owner,
3550            query_task_attempt,
3551            typed_result.into_json()?,
3552            result_envelope,
3553        )
3554        .await
3555    }
3556
3557    async fn complete_query_task_with_envelope(
3558        &self,
3559        query_task_id: &str,
3560        lease_owner: &str,
3561        query_task_attempt: u64,
3562        result: Value,
3563        result_envelope: Value,
3564    ) -> Result<Value> {
3565        let body = json!({
3566            "lease_owner": lease_owner,
3567            "query_task_attempt": query_task_attempt,
3568            "result": result,
3569            "result_envelope": result_envelope,
3570        });
3571        let path = format!("/worker/query-tasks/{query_task_id}/complete");
3572        let response = self
3573            .request_json(
3574                reqwest::Method::POST,
3575                &path,
3576                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3577                Some(&body),
3578            )
3579            .await;
3580        query_task_response(response)
3581    }
3582
3583    /// Report a stable machine-readable query-task failure.
3584    pub async fn fail_query_task(
3585        &self,
3586        query_task_id: &str,
3587        lease_owner: &str,
3588        query_task_attempt: u64,
3589        message: impl Into<String>,
3590        reason: impl Into<String>,
3591        failure_type: impl Into<String>,
3592    ) -> Result<Value> {
3593        let body = json!({
3594            "lease_owner": lease_owner,
3595            "query_task_attempt": query_task_attempt,
3596            "failure": {
3597                "message": message.into(),
3598                "reason": reason.into(),
3599                "type": failure_type.into(),
3600            }
3601        });
3602        let path = format!("/worker/query-tasks/{query_task_id}/fail");
3603        let response = self
3604            .request_json(
3605                reqwest::Method::POST,
3606                &path,
3607                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3608                Some(&body),
3609            )
3610            .await;
3611        query_task_response(response)
3612    }
3613
3614    pub async fn heartbeat_worker(
3615        &self,
3616        worker_id: &str,
3617        workflow_available: usize,
3618        activity_available: usize,
3619    ) -> Result<Value> {
3620        let body = json!({
3621            "worker_id": worker_id,
3622            "task_slots": {
3623                "workflow_available": workflow_available,
3624                "activity_available": activity_available
3625            },
3626            "process_metrics": {
3627                "process_id": std::process::id(),
3628                "process_uptime_seconds": 0
3629            }
3630        });
3631
3632        self.request_json(
3633            reqwest::Method::POST,
3634            "/worker/heartbeat",
3635            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3636            Some(&body),
3637        )
3638        .await
3639    }
3640
3641    pub async fn poll_workflow_task(
3642        &self,
3643        worker_id: &str,
3644        task_queue: &str,
3645        timeout: Duration,
3646    ) -> Result<Option<WorkflowTask>> {
3647        Ok(self
3648            .poll_workflow_task_response(worker_id, task_queue, timeout)
3649            .await?
3650            .task)
3651    }
3652
3653    pub async fn poll_workflow_task_response(
3654        &self,
3655        worker_id: &str,
3656        task_queue: &str,
3657        timeout: Duration,
3658    ) -> Result<PollWorkflowTaskResponse> {
3659        let poll_request_id = unique_request_id("rust-workflow-poll");
3660        self.poll_workflow_task_response_with_request_id(
3661            worker_id,
3662            task_queue,
3663            timeout,
3664            &poll_request_id,
3665            1,
3666        )
3667        .await
3668    }
3669
3670    async fn poll_workflow_task_response_with_request_id(
3671        &self,
3672        worker_id: &str,
3673        task_queue: &str,
3674        timeout: Duration,
3675        poll_request_id: &str,
3676        transport_retries: usize,
3677    ) -> Result<PollWorkflowTaskResponse> {
3678        let body = json!({
3679            "worker_id": worker_id,
3680            "task_queue": task_queue,
3681            "poll_request_id": poll_request_id,
3682            "timeout_seconds": long_poll_timeout_seconds(timeout),
3683        });
3684        let mut data: PollWorkflowTaskResponse = self
3685            .poll_request_json(
3686                "/worker/workflow-tasks/poll",
3687                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3688                &body,
3689                timeout + Duration::from_secs(5),
3690                transport_retries,
3691            )
3692            .await?;
3693
3694        if let Some(task) = data.task.as_mut() {
3695            self.fetch_remaining_workflow_history(worker_id, task)
3696                .await?;
3697        }
3698
3699        Ok(data)
3700    }
3701
3702    async fn fetch_remaining_workflow_history(
3703        &self,
3704        worker_id: &str,
3705        task: &mut WorkflowTask,
3706    ) -> Result<()> {
3707        let mut next_token = task.next_history_page_token.clone();
3708
3709        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3710            let lease_owner = task
3711                .lease_owner
3712                .clone()
3713                .unwrap_or_else(|| worker_id.to_string());
3714            let page = self
3715                .workflow_task_history_page(
3716                    &task.task_id,
3717                    &lease_owner,
3718                    task.workflow_task_attempt,
3719                    &token,
3720                )
3721                .await?;
3722
3723            task.append_history_page(page);
3724
3725            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3726                return Err(Error::Codec(
3727                    "workflow history pagination returned the same page token".to_string(),
3728                ));
3729            }
3730
3731            next_token = task.next_history_page_token.clone();
3732        }
3733
3734        Ok(())
3735    }
3736
3737    async fn workflow_task_history_page(
3738        &self,
3739        task_id: &str,
3740        lease_owner: &str,
3741        workflow_task_attempt: u64,
3742        next_history_page_token: &str,
3743    ) -> Result<WorkflowTaskHistoryPage> {
3744        let body = json!({
3745            "lease_owner": lease_owner,
3746            "workflow_task_attempt": workflow_task_attempt,
3747            "next_history_page_token": next_history_page_token
3748        });
3749        let path = format!("/worker/workflow-tasks/{task_id}/history");
3750
3751        self.request_json(
3752            reqwest::Method::POST,
3753            &path,
3754            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3755            Some(&body),
3756        )
3757        .await
3758    }
3759
3760    pub async fn complete_workflow_task(
3761        &self,
3762        task_id: &str,
3763        lease_owner: &str,
3764        workflow_task_attempt: u64,
3765        commands: Vec<Value>,
3766    ) -> Result<Value> {
3767        self.complete_workflow_task_with_message_streams(
3768            task_id,
3769            lease_owner,
3770            workflow_task_attempt,
3771            commands,
3772            Vec::new(),
3773            Vec::new(),
3774        )
3775        .await
3776    }
3777
3778    async fn complete_workflow_task_with_message_streams(
3779        &self,
3780        task_id: &str,
3781        lease_owner: &str,
3782        workflow_task_attempt: u64,
3783        commands: Vec<Value>,
3784        message_stream_cursors: Vec<Value>,
3785        message_stream_waits: Vec<Value>,
3786    ) -> Result<Value> {
3787        validate_workflow_task_commands(&commands)?;
3788        let has_message_stream_metadata =
3789            !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3790        if has_message_stream_metadata
3791            && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3792        {
3793            return Err(Error::Codec(
3794                "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3795                    .to_string(),
3796            ));
3797        }
3798        let protocol_version = workflow_completion_protocol_version_with_message_streams(
3799            &commands,
3800            has_message_stream_metadata,
3801        );
3802        let mut body = json!({
3803            "lease_owner": lease_owner,
3804            "workflow_task_attempt": workflow_task_attempt,
3805            "commands": commands
3806        });
3807        if !message_stream_cursors.is_empty() {
3808            body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3809        }
3810        if !message_stream_waits.is_empty() {
3811            body["message_stream_waits"] = Value::Array(message_stream_waits);
3812        }
3813        let path = format!("/worker/workflow-tasks/{task_id}/complete");
3814        self.request_json(
3815            reqwest::Method::POST,
3816            &path,
3817            RequestProtocol::Worker(protocol_version),
3818            Some(&body),
3819        )
3820        .await
3821    }
3822
3823    pub async fn fail_workflow_task(
3824        &self,
3825        task_id: &str,
3826        lease_owner: &str,
3827        workflow_task_attempt: u64,
3828        message: impl Into<String>,
3829    ) -> Result<Value> {
3830        self.fail_workflow_task_with_type(
3831            task_id,
3832            lease_owner,
3833            workflow_task_attempt,
3834            message,
3835            "RustWorkflowTaskFailure",
3836        )
3837        .await
3838    }
3839
3840    async fn fail_workflow_task_with_type(
3841        &self,
3842        task_id: &str,
3843        lease_owner: &str,
3844        workflow_task_attempt: u64,
3845        message: impl Into<String>,
3846        failure_type: &str,
3847    ) -> Result<Value> {
3848        let body = json!({
3849            "lease_owner": lease_owner,
3850            "workflow_task_attempt": workflow_task_attempt,
3851            "failure": {
3852                "message": message.into(),
3853                "type": failure_type
3854            }
3855        });
3856        let path = format!("/worker/workflow-tasks/{task_id}/fail");
3857        self.request_json(
3858            reqwest::Method::POST,
3859            &path,
3860            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3861            Some(&body),
3862        )
3863        .await
3864    }
3865
3866    pub async fn poll_activity_task(
3867        &self,
3868        worker_id: &str,
3869        task_queue: &str,
3870        timeout: Duration,
3871    ) -> Result<Option<ActivityTask>> {
3872        Ok(self
3873            .poll_activity_task_response(worker_id, task_queue, timeout)
3874            .await?
3875            .task)
3876    }
3877
3878    /// Poll an activity task while preserving server stop and drain metadata.
3879    pub async fn poll_activity_task_response(
3880        &self,
3881        worker_id: &str,
3882        task_queue: &str,
3883        timeout: Duration,
3884    ) -> Result<PollActivityTaskResponse> {
3885        let poll_request_id = unique_request_id("rust-activity-poll");
3886        self.poll_activity_task_response_with_request_id(
3887            worker_id,
3888            task_queue,
3889            timeout,
3890            &poll_request_id,
3891            1,
3892        )
3893        .await
3894    }
3895
3896    async fn poll_activity_task_response_with_request_id(
3897        &self,
3898        worker_id: &str,
3899        task_queue: &str,
3900        timeout: Duration,
3901        poll_request_id: &str,
3902        transport_retries: usize,
3903    ) -> Result<PollActivityTaskResponse> {
3904        let body = json!({
3905            "worker_id": worker_id,
3906            "task_queue": task_queue,
3907            "poll_request_id": poll_request_id,
3908            "timeout_seconds": long_poll_timeout_seconds(timeout),
3909        });
3910        let data: PollActivityTaskResponse = self
3911            .poll_request_json(
3912                "/worker/activity-tasks/poll",
3913                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3914                &body,
3915                timeout + Duration::from_secs(5),
3916                transport_retries,
3917            )
3918            .await?;
3919        Ok(data)
3920    }
3921
3922    pub async fn complete_activity_task<T: Serialize>(
3923        &self,
3924        task_id: &str,
3925        activity_attempt_id: &str,
3926        lease_owner: &str,
3927        result: T,
3928        codec: &str,
3929    ) -> Result<Value> {
3930        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3931        let body = json!({
3932            "activity_attempt_id": activity_attempt_id,
3933            "lease_owner": lease_owner,
3934            "result": result
3935        });
3936        let path = format!("/worker/activity-tasks/{task_id}/complete");
3937        activity_task_response(
3938            self.request_json(
3939                reqwest::Method::POST,
3940                &path,
3941                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3942                Some(&body),
3943            )
3944            .await,
3945            "complete",
3946            task_id,
3947            activity_attempt_id,
3948        )
3949    }
3950
3951    pub async fn fail_activity_task(
3952        &self,
3953        task_id: &str,
3954        activity_attempt_id: &str,
3955        lease_owner: &str,
3956        message: impl Into<String>,
3957        non_retryable: bool,
3958    ) -> Result<Value> {
3959        let body = json!({
3960            "activity_attempt_id": activity_attempt_id,
3961            "lease_owner": lease_owner,
3962            "failure": {
3963                "message": message.into(),
3964                "type": "RustActivityFailure",
3965                "non_retryable": non_retryable
3966            }
3967        });
3968        let path = format!("/worker/activity-tasks/{task_id}/fail");
3969        activity_task_response(
3970            self.request_json(
3971                reqwest::Method::POST,
3972                &path,
3973                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3974                Some(&body),
3975            )
3976            .await,
3977            "fail",
3978            task_id,
3979            activity_attempt_id,
3980        )
3981    }
3982
3983    pub async fn heartbeat_activity_task<T: Serialize>(
3984        &self,
3985        task_id: &str,
3986        activity_attempt_id: &str,
3987        lease_owner: &str,
3988        details: T,
3989    ) -> Result<ActivityHeartbeatResponse> {
3990        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3991        let body = json!({
3992            "activity_attempt_id": activity_attempt_id,
3993            "lease_owner": lease_owner,
3994            "details": details
3995        });
3996        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
3997        activity_task_response(
3998            self.request_json(
3999                reqwest::Method::POST,
4000                &path,
4001                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4002                Some(&body),
4003            )
4004            .await,
4005            "heartbeat",
4006            task_id,
4007            activity_attempt_id,
4008        )
4009    }
4010
4011    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4012        &self,
4013        method: reqwest::Method,
4014        path: &str,
4015        protocol: RequestProtocol,
4016        body: Option<&B>,
4017    ) -> Result<T> {
4018        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4019            .await
4020    }
4021
4022    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4023        &self,
4024        method: reqwest::Method,
4025        path: &str,
4026        protocol: RequestProtocol,
4027        body: Option<&B>,
4028        timeout: Duration,
4029    ) -> Result<T> {
4030        let auth_token = self.auth_token(protocol)?;
4031        let mut request = self
4032            .http
4033            .request(method, format!("{}/api{}", self.base_url, path))
4034            .timeout(timeout)
4035            .header(reqwest::header::ACCEPT, "application/json")
4036            .header(reqwest::header::CONTENT_TYPE, "application/json")
4037            .header("X-Namespace", &self.namespace);
4038
4039        match protocol {
4040            RequestProtocol::Worker(version) => {
4041                request = request.header("X-Durable-Workflow-Protocol-Version", version);
4042            }
4043            RequestProtocol::ControlPlane => {
4044                request = request.header(
4045                    "X-Durable-Workflow-Control-Plane-Version",
4046                    CONTROL_PLANE_VERSION,
4047                );
4048            }
4049        }
4050
4051        if let Some(token) = auth_token {
4052            request = request.bearer_auth(token);
4053        }
4054
4055        if let Some(body) = body {
4056            request = request.json(body);
4057        }
4058
4059        let request = request.build()?;
4060        let admission = self
4061            .worker_storage_admission
4062            .as_ref()
4063            .filter(|_| matches!(protocol, RequestProtocol::Worker(_)));
4064        let poll_request_id = path.ends_with("/poll").then(|| {
4065            request
4066                .body()
4067                .and_then(reqwest::Body::as_bytes)
4068                .and_then(|body| serde_json::from_slice::<Value>(body).ok())
4069                .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4070                .unwrap_or_default()
4071        });
4072        let mut storage_retries = 0_usize;
4073
4074        loop {
4075            // Keep the serialized body and lease/poll identity across admission pauses.
4076            let response = self
4077                .http
4078                .execute(request.try_clone().ok_or_else(|| {
4079                    Error::WorkerLoop("worker request body cannot be retried".to_string())
4080                })?)
4081                .await?;
4082            let status = response.status();
4083            let bytes = response.bytes().await?;
4084
4085            if !status.is_success() {
4086                let body = String::from_utf8_lossy(&bytes).to_string();
4087                if let Some(protocol) = protocol_failure(status, &body) {
4088                    return Err(Error::Protocol(protocol));
4089                }
4090                let error = Error::Http { status, body };
4091                if let Some(admission) = admission {
4092                    if let Some(advertised_delay) =
4093                        worker_storage_admission_retry_after(&error, poll_request_id.as_deref())
4094                    {
4095                        storage_retries = storage_retries.saturating_add(1);
4096                        let delay = worker_retry_delay(admission.policy, storage_retries)
4097                            .max(advertised_delay)
4098                            .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4099                        let deadline = tokio::time::Instant::now() + delay;
4100                        loop {
4101                            if admission.stop.load(Ordering::SeqCst) {
4102                                return Err(error);
4103                            }
4104                            let remaining =
4105                                deadline.saturating_duration_since(tokio::time::Instant::now());
4106                            if remaining.is_zero() {
4107                                break;
4108                            }
4109                            tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4110                        }
4111                        continue;
4112                    }
4113                }
4114                return Err(error);
4115            }
4116
4117            if bytes.is_empty() {
4118                return Ok(serde_json::from_value(Value::Null)?);
4119            }
4120
4121            return Ok(serde_json::from_slice(&bytes)?);
4122        }
4123    }
4124
4125    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4126        &self,
4127        path: &str,
4128        protocol: RequestProtocol,
4129        body: &B,
4130        timeout: Duration,
4131        max_retries: usize,
4132    ) -> Result<T> {
4133        let mut retries = 0;
4134
4135        loop {
4136            let response = self
4137                .request_json_with_timeout(
4138                    reqwest::Method::POST,
4139                    path,
4140                    protocol,
4141                    Some(body),
4142                    timeout,
4143                )
4144                .await;
4145
4146            match response {
4147                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4148                response => return worker_poll_response(response),
4149            }
4150        }
4151    }
4152
4153    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4154        match protocol {
4155            RequestProtocol::Worker(_) => {
4156                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4157                    return Ok(Some(token));
4158                }
4159                if self.control_token.is_some() {
4160                    return Err(Error::MissingRoleCredentials {
4161                        role: "worker",
4162                        opposite_role: "control",
4163                    });
4164                }
4165                Ok(None)
4166            }
4167            RequestProtocol::ControlPlane => {
4168                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4169                    return Ok(Some(token));
4170                }
4171                if self.worker_token.is_some() {
4172                    return Err(Error::MissingRoleCredentials {
4173                        role: "control",
4174                        opposite_role: "worker",
4175                    });
4176                }
4177                Ok(None)
4178            }
4179        }
4180    }
4181}
4182
4183fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4184    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4185    let reason = body
4186        .get("reason")
4187        .and_then(Value::as_str)
4188        .unwrap_or("query_rejected")
4189        .to_string();
4190    let message = body
4191        .get("message")
4192        .or_else(|| body.get("error"))
4193        .and_then(Value::as_str)
4194        .unwrap_or("workflow query was rejected")
4195        .to_string();
4196
4197    QueryFailure {
4198        status: status.as_u16(),
4199        reason,
4200        message,
4201        body,
4202    }
4203}
4204
4205fn workflow_command_result(
4206    command: WorkflowCommandKind,
4207    data: Value,
4208    workflow_id: &str,
4209    run_id: Option<&str>,
4210) -> WorkflowCommandResult {
4211    WorkflowCommandResult {
4212        command,
4213        workflow_id: data
4214            .get("workflow_id")
4215            .and_then(Value::as_str)
4216            .unwrap_or(workflow_id)
4217            .to_string(),
4218        run_id: data
4219            .get("run_id")
4220            .and_then(Value::as_str)
4221            .or(run_id)
4222            .map(str::to_string),
4223        outcome: data
4224            .get("outcome")
4225            .and_then(Value::as_str)
4226            .map(str::to_string),
4227        reason: data
4228            .get("reason")
4229            .and_then(Value::as_str)
4230            .map(str::to_string),
4231        command_status: data
4232            .get("command_status")
4233            .and_then(Value::as_str)
4234            .map(str::to_string),
4235        raw: data,
4236    }
4237}
4238
4239fn workflow_command_rejection(
4240    command: WorkflowCommandKind,
4241    status: reqwest::StatusCode,
4242    raw_body: String,
4243    workflow_id: &str,
4244    run_id: Option<&str>,
4245) -> WorkflowCommandRejection {
4246    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4247    WorkflowCommandRejection {
4248        command,
4249        status: status.as_u16(),
4250        reason: body
4251            .get("reason")
4252            .and_then(Value::as_str)
4253            .unwrap_or("workflow_command_rejected")
4254            .to_string(),
4255        message: body
4256            .get("message")
4257            .or_else(|| body.get("error"))
4258            .and_then(Value::as_str)
4259            .unwrap_or("workflow lifecycle command was rejected")
4260            .to_string(),
4261        workflow_id: body
4262            .get("workflow_id")
4263            .and_then(Value::as_str)
4264            .unwrap_or(workflow_id)
4265            .to_string(),
4266        run_id: body
4267            .get("run_id")
4268            .and_then(Value::as_str)
4269            .or(run_id)
4270            .map(str::to_string),
4271        target_scope: body
4272            .get("target_scope")
4273            .and_then(Value::as_str)
4274            .map(str::to_string),
4275        body,
4276    }
4277}
4278
4279fn query_task_response(response: Result<Value>) -> Result<Value> {
4280    match response {
4281        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4282        response => response,
4283    }
4284}
4285
4286fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4287    match response {
4288        Err(Error::Http { status, body })
4289            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4290        {
4291            Ok(serde_json::from_str(&body)?)
4292        }
4293        response => response,
4294    }
4295}
4296
4297fn worker_poll_body_is_stop(body: &str) -> bool {
4298    serde_json::from_str::<Value>(body)
4299        .ok()
4300        .is_some_and(|body| {
4301            worker_poll_is_stop(
4302                body.get("poll_status").and_then(Value::as_str),
4303                body.get("reason").and_then(Value::as_str),
4304            )
4305        })
4306}
4307
4308fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4309    matches!(poll_status, Some("draining" | "stopped"))
4310        || matches!(reason, Some("worker_draining" | "worker_stopped"))
4311}
4312
4313fn query_task_rejection_is_final(error: &Error) -> bool {
4314    matches!(
4315        error,
4316        Error::QueryFailed(failure)
4317            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4318    )
4319}
4320
4321fn activity_task_response<T>(
4322    response: Result<T>,
4323    operation: &str,
4324    task_id: &str,
4325    activity_attempt_id: &str,
4326) -> Result<T> {
4327    match response {
4328        Err(Error::Http { status, body }) => {
4329            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4330            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4331                operation: operation.to_string(),
4332                status: status.as_u16(),
4333                reason: body
4334                    .get("reason")
4335                    .and_then(Value::as_str)
4336                    .unwrap_or("activity_task_rejected")
4337                    .to_string(),
4338                task_id: body
4339                    .get("task_id")
4340                    .and_then(Value::as_str)
4341                    .unwrap_or(task_id)
4342                    .to_string(),
4343                activity_attempt_id: body
4344                    .get("activity_attempt_id")
4345                    .and_then(Value::as_str)
4346                    .unwrap_or(activity_attempt_id)
4347                    .to_string(),
4348                cancel_requested: body
4349                    .get("cancel_requested")
4350                    .and_then(Value::as_bool)
4351                    .unwrap_or(false),
4352                can_continue: body.get("can_continue").and_then(Value::as_bool),
4353                run_closed_reason: body
4354                    .get("run_closed_reason")
4355                    .and_then(Value::as_str)
4356                    .map(str::to_string),
4357                body,
4358            }))
4359        }
4360        response => response,
4361    }
4362}
4363
4364fn activity_task_rejection_is_final(error: &Error) -> bool {
4365    matches!(
4366        error,
4367        Error::ActivityTaskRejected(rejection)
4368            if matches!(
4369                rejection.reason.as_str(),
4370                "run_cancelled"
4371                    | "run_terminated"
4372                    | "attempt_closed"
4373                    | "stale_attempt"
4374                    | "activity_cancelled"
4375                    | "task_cancelled"
4376                    | "run_closed"
4377                    | "activity_not_running"
4378                    | "attempt_not_found"
4379            )
4380    )
4381}
4382
4383fn workflow_task_completion_is_terminal_timeout(
4384    error: &Error,
4385    task_id: &str,
4386    workflow_task_attempt: u64,
4387    run_id: Option<&str>,
4388) -> bool {
4389    let Error::Http { status, body } = error else {
4390        return false;
4391    };
4392    if *status != reqwest::StatusCode::CONFLICT {
4393        return false;
4394    }
4395
4396    let Some(run_id) = run_id else {
4397        return false;
4398    };
4399    let Ok(body) = serde_json::from_str::<Value>(body) else {
4400        return false;
4401    };
4402
4403    body.get("recorded").and_then(Value::as_bool) == Some(false)
4404        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4405        && body.get("run_status").and_then(Value::as_str) == Some("failed")
4406        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4407        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4408        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4409}
4410
4411fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4412    let body: Value = serde_json::from_str(raw_body).ok()?;
4413    let reason = body.get("reason")?.as_str()?;
4414    if !matches!(
4415        reason,
4416        "missing_protocol_version"
4417            | "unsupported_protocol_version"
4418            | "missing_control_plane_version"
4419            | "unsupported_control_plane_version"
4420    ) {
4421        return None;
4422    }
4423
4424    Some(ProtocolFailure {
4425        status: status.as_u16(),
4426        reason: reason.to_string(),
4427        message: body
4428            .get("message")
4429            .or_else(|| body.get("error"))
4430            .and_then(Value::as_str)
4431            .unwrap_or("protocol version rejected")
4432            .to_string(),
4433        supported_version: body
4434            .get("supported_version")
4435            .and_then(Value::as_str)
4436            .map(str::to_string),
4437        requested_version: body
4438            .get("requested_version")
4439            .and_then(Value::as_str)
4440            .map(str::to_string),
4441        body,
4442    })
4443}
4444
4445fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4446    timeout
4447        .as_secs()
4448        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4449        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4450}
4451
4452fn worker_operation_is_retryable(error: &Error) -> bool {
4453    if worker_poll_capacity_retry_after(error).is_some()
4454        || worker_storage_admission_body(error).is_some()
4455        || worker_operation_is_explicitly_non_retryable(error)
4456    {
4457        return false;
4458    }
4459
4460    match error {
4461        Error::Transport(error) => {
4462            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4463        }
4464        Error::Http { status, .. } => {
4465            matches!(
4466                *status,
4467                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4468            ) || status.is_server_error()
4469        }
4470        _ => false,
4471    }
4472}
4473
4474fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4475    let body: Value = match error {
4476        Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4477        Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4478        _ => return None,
4479    };
4480    matches!(
4481        body.get("reason").and_then(Value::as_str),
4482        Some("storage_pressure" | "storage_admission_unavailable")
4483    )
4484    .then_some(body)
4485}
4486
4487fn worker_storage_admission_retry_after(
4488    error: &Error,
4489    poll_request_id: Option<&str>,
4490) -> Option<Duration> {
4491    let Error::Http { status, .. } = error else {
4492        return None;
4493    };
4494    let body = worker_storage_admission_body(error)?;
4495    let delay = body.get("retry_after_seconds")?.as_u64()?;
4496    if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4497        || delay == 0
4498        || body.get("retryable") != Some(&Value::Bool(true))
4499        || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4500        || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4501        || body
4502            .get("request_admitted")
4503            .is_some_and(|admitted| admitted != &Value::Bool(false))
4504    {
4505        return None;
4506    }
4507    match poll_request_id {
4508        Some(id) => {
4509            if id.is_empty()
4510                || body.get("task") != Some(&Value::Null)
4511                || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4512                || body.get("poll_status") != body.get("reason")
4513                || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4514                || body.get("claim_admitted") != Some(&Value::Bool(false))
4515            {
4516                return None;
4517            }
4518        }
4519        None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4520        None => {}
4521    }
4522    Some(Duration::from_secs(delay))
4523}
4524
4525fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4526    let Error::Http { body, .. } = error else {
4527        return false;
4528    };
4529
4530    serde_json::from_str::<Value>(body)
4531        .ok()
4532        .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4533        == Some(false)
4534}
4535
4536fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4537    let Error::Http { status, body } = error else {
4538        return None;
4539    };
4540    if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4541        return None;
4542    }
4543
4544    let body = serde_json::from_str::<Value>(body).ok()?;
4545    let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4546        == Some("long_poll_capacity_exhausted")
4547        || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4548    if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4549        return None;
4550    }
4551
4552    Some(Duration::from_secs(
4553        body.get("retry_after_seconds")
4554            .and_then(Value::as_u64)
4555            .unwrap_or_default(),
4556    ))
4557}
4558
4559fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4560    let exponent = retry.saturating_sub(1).min(31) as u32;
4561    policy
4562        .initial_backoff
4563        .saturating_mul(1_u32 << exponent)
4564        .min(policy.max_backoff)
4565}
4566
4567#[derive(Debug)]
4568pub struct ClientBuilder {
4569    base_url: String,
4570    token: Option<String>,
4571    control_token: Option<String>,
4572    worker_token: Option<String>,
4573    namespace: String,
4574    timeout: Duration,
4575}
4576
4577impl ClientBuilder {
4578    pub fn token(mut self, token: Option<String>) -> Self {
4579        self.token = token;
4580        self
4581    }
4582
4583    pub fn control_token(mut self, token: Option<String>) -> Self {
4584        self.control_token = token;
4585        self
4586    }
4587
4588    pub fn worker_token(mut self, token: Option<String>) -> Self {
4589        self.worker_token = token;
4590        self
4591    }
4592
4593    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4594        self.namespace = namespace.into();
4595        self
4596    }
4597
4598    pub fn timeout(mut self, timeout: Duration) -> Self {
4599        self.timeout = timeout;
4600        self
4601    }
4602
4603    pub fn build(self) -> Result<Client> {
4604        let base_url = self.base_url.trim_end_matches('/').to_string();
4605        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4606            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4607            .unwrap_or_else(|_| base_url.ends_with("/api"));
4608
4609        if has_sdk_api_suffix {
4610            return Err(Error::InvalidBaseUrl);
4611        }
4612
4613        Ok(Client {
4614            http: reqwest::Client::builder().timeout(self.timeout).build()?,
4615            base_url,
4616            token: self.token,
4617            control_token: self.control_token,
4618            worker_token: self.worker_token,
4619            namespace: self.namespace,
4620            worker_storage_admission: None,
4621        })
4622    }
4623}
4624
4625#[derive(Clone, Debug)]
4626pub struct WorkflowHandle {
4627    client: Client,
4628    pub workflow_id: String,
4629    pub run_id: Option<String>,
4630    pub workflow_type: String,
4631}
4632
4633impl WorkflowHandle {
4634    /// Describe whichever run is current for this stable workflow instance.
4635    pub async fn describe(&self) -> Result<WorkflowDescription> {
4636        self.client.describe_workflow(&self.workflow_id).await
4637    }
4638
4639    /// Describe the run identity originally selected by this handle.
4640    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4641        let run_id = self.run_id.as_deref().ok_or_else(|| {
4642            Error::Codec("run_id is required for selected-run description".to_string())
4643        })?;
4644        self.client
4645            .describe_workflow_run(&self.workflow_id, run_id)
4646            .await
4647    }
4648
4649    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4650        self.client
4651            .signal_workflow(&self.workflow_id, signal_name, input)
4652            .await
4653    }
4654
4655    pub async fn append_message<T: Serialize>(
4656        &self,
4657        stream_name: &str,
4658        message_id: &str,
4659        input: T,
4660    ) -> Result<Value> {
4661        self.client
4662            .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4663            .await
4664    }
4665
4666    /// Signal only if this handle's selected run is still current.
4667    pub async fn signal_selected_run<T: Serialize>(
4668        &self,
4669        signal_name: &str,
4670        input: T,
4671    ) -> Result<Value> {
4672        let run_id = self.run_id.as_deref().ok_or_else(|| {
4673            Error::Codec("run_id is required for selected-run signaling".to_string())
4674        })?;
4675        self.client
4676            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4677            .await
4678    }
4679
4680    /// Request cooperative cancellation of whichever run is current.
4681    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4682        self.client
4683            .cancel_workflow(&self.workflow_id, options)
4684            .await
4685    }
4686
4687    /// Request cancellation only if this handle's selected run is still current.
4688    pub async fn cancel_selected_run(
4689        &self,
4690        options: WorkflowCommandOptions,
4691    ) -> Result<WorkflowCommandResult> {
4692        let run_id = self.run_id.as_deref().ok_or_else(|| {
4693            Error::Codec("run_id is required for selected-run cancellation".to_string())
4694        })?;
4695        self.client
4696            .cancel_workflow_run(&self.workflow_id, run_id, options)
4697            .await
4698    }
4699
4700    /// Forcefully terminate whichever run is current.
4701    pub async fn terminate(
4702        &self,
4703        options: WorkflowCommandOptions,
4704    ) -> Result<WorkflowCommandResult> {
4705        self.client
4706            .terminate_workflow(&self.workflow_id, options)
4707            .await
4708    }
4709
4710    /// Terminate only if this handle's selected run is still current.
4711    pub async fn terminate_selected_run(
4712        &self,
4713        options: WorkflowCommandOptions,
4714    ) -> Result<WorkflowCommandResult> {
4715        let run_id = self.run_id.as_deref().ok_or_else(|| {
4716            Error::Codec("run_id is required for selected-run termination".to_string())
4717        })?;
4718        self.client
4719            .terminate_workflow_run(&self.workflow_id, run_id, options)
4720            .await
4721    }
4722
4723    /// Execute a named, read-only query against this workflow.
4724    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4725        self.client
4726            .query_workflow(&self.workflow_id, query_name, input)
4727            .await
4728    }
4729
4730    pub async fn query_avro_value<T: Serialize>(
4731        &self,
4732        query_name: &str,
4733        input: T,
4734    ) -> Result<AvroValue> {
4735        self.client
4736            .query_workflow_avro_value(&self.workflow_id, query_name, input)
4737            .await
4738    }
4739
4740    pub async fn update<T: Serialize>(
4741        &self,
4742        update_name: &str,
4743        input: T,
4744        request_id: Option<&str>,
4745    ) -> Result<Value> {
4746        self.client
4747            .update_workflow(&self.workflow_id, update_name, input, request_id)
4748            .await
4749    }
4750
4751    pub async fn update_avro_value<T: Serialize>(
4752        &self,
4753        update_name: &str,
4754        input: T,
4755        request_id: Option<&str>,
4756    ) -> Result<AvroValue> {
4757        self.client
4758            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4759            .await
4760    }
4761
4762    /// Query only if this handle's selected run is still current.
4763    pub async fn query_selected_run<T: Serialize>(
4764        &self,
4765        query_name: &str,
4766        input: T,
4767    ) -> Result<Value> {
4768        let run_id = self
4769            .run_id
4770            .as_deref()
4771            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4772        self.client
4773            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4774            .await
4775    }
4776
4777    /// Await the final terminal outcome of the current continue-as-new chain.
4778    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4779        self.result_target(options, None).await
4780    }
4781
4782    /// Await the final result without projecting Avro bytes through JSON.
4783    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4784        self.result_avro_value_target(options, None).await
4785    }
4786
4787    /// Await the final result and decode it into a Serde application type.
4788    pub async fn result_typed<T: DeserializeOwned>(
4789        &self,
4790        options: WorkflowResultOptions,
4791    ) -> Result<T> {
4792        let result = self.result_avro_value(options).await?;
4793        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4794    }
4795
4796    /// Await only the run identity originally selected by this handle.
4797    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4798        let run_id = self.run_id.as_deref().ok_or_else(|| {
4799            Error::Codec("run_id is required for selected-run result".to_string())
4800        })?;
4801        self.result_target(options, Some(run_id)).await
4802    }
4803
4804    /// Await the selected run's result on the lossless Avro Value surface.
4805    pub async fn result_selected_run_avro_value(
4806        &self,
4807        options: WorkflowResultOptions,
4808    ) -> Result<AvroValue> {
4809        let run_id = self.run_id.as_deref().ok_or_else(|| {
4810            Error::Codec("run_id is required for selected-run result".to_string())
4811        })?;
4812        self.result_avro_value_target(options, Some(run_id)).await
4813    }
4814
4815    /// Await the selected run and decode its result into a Serde type.
4816    pub async fn result_selected_run_typed<T: DeserializeOwned>(
4817        &self,
4818        options: WorkflowResultOptions,
4819    ) -> Result<T> {
4820        let result = self.result_selected_run_avro_value(options).await?;
4821        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4822    }
4823
4824    async fn result_avro_value_target(
4825        &self,
4826        options: WorkflowResultOptions,
4827        selected_run_id: Option<&str>,
4828    ) -> Result<AvroValue> {
4829        let started = Instant::now();
4830
4831        loop {
4832            let description = match selected_run_id {
4833                Some(run_id) => {
4834                    self.client
4835                        .describe_workflow_run(&self.workflow_id, run_id)
4836                        .await?
4837                }
4838                None => self.describe().await?,
4839            };
4840            if description.is_completed() {
4841                return description.output_avro_value.ok_or_else(|| {
4842                    Error::Codec(
4843                        "missing_payload_envelope: typed workflow result requires output_envelope"
4844                            .to_string(),
4845                    )
4846                });
4847            }
4848            if description.is_terminal() {
4849                let outcome =
4850                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4851                return Err(match outcome.kind {
4852                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4853                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4854                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4855                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4856                });
4857            }
4858            if started.elapsed() >= options.timeout {
4859                return Err(Error::Timeout);
4860            }
4861            tokio::time::sleep(options.poll_interval).await;
4862        }
4863    }
4864
4865    async fn result_target(
4866        &self,
4867        options: WorkflowResultOptions,
4868        selected_run_id: Option<&str>,
4869    ) -> Result<Value> {
4870        let started = Instant::now();
4871
4872        loop {
4873            let description = match selected_run_id {
4874                Some(run_id) => {
4875                    self.client
4876                        .describe_workflow_run(&self.workflow_id, run_id)
4877                        .await?
4878                }
4879                None => self.describe().await?,
4880            };
4881            if description.is_completed() {
4882                return Ok(description.output.unwrap_or(Value::Null));
4883            }
4884
4885            if description.is_terminal() {
4886                let outcome =
4887                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4888                return Err(match outcome.kind {
4889                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4890                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4891                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4892                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4893                });
4894            }
4895
4896            if started.elapsed() >= options.timeout {
4897                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4898                    kind: WorkflowTerminalKind::TimedOut,
4899                    workflow_id: description
4900                        .workflow_id
4901                        .clone()
4902                        .unwrap_or_else(|| self.workflow_id.clone()),
4903                    run_id: description
4904                        .run_id
4905                        .clone()
4906                        .or_else(|| selected_run_id.map(str::to_string)),
4907                    reason: "result_wait_timeout".to_string(),
4908                    failure_category: Some("client_timeout".to_string()),
4909                    failure_id: None,
4910                    exception_type: None,
4911                    exception_class: None,
4912                    non_retryable: None,
4913                    message: Some(format!(
4914                        "workflow result was not terminal within {:?}",
4915                        options.timeout
4916                    )),
4917                    exception: None,
4918                    raw: description.raw_value(),
4919                }));
4920            }
4921
4922            tokio::time::sleep(options.poll_interval).await;
4923        }
4924    }
4925}
4926
4927#[derive(Clone, Copy, Debug)]
4928pub struct WorkflowResultOptions {
4929    pub poll_interval: Duration,
4930    pub timeout: Duration,
4931}
4932
4933impl Default for WorkflowResultOptions {
4934    fn default() -> Self {
4935        Self {
4936            poll_interval: Duration::from_millis(500),
4937            timeout: Duration::from_secs(30),
4938        }
4939    }
4940}
4941
4942#[derive(Clone, Debug, Deserialize)]
4943pub struct WorkflowDescription {
4944    pub workflow_id: Option<String>,
4945    pub run_id: Option<String>,
4946    pub workflow_type: Option<String>,
4947    pub status: Option<String>,
4948    #[serde(default)]
4949    pub closed_reason: Option<String>,
4950    #[serde(default)]
4951    pub error: Option<String>,
4952    #[serde(default)]
4953    pub failure: Option<Value>,
4954    #[serde(default)]
4955    pub exception: Option<Value>,
4956    #[serde(default)]
4957    pub failures: Vec<Value>,
4958    #[serde(default)]
4959    pub output: Option<Value>,
4960    #[serde(default)]
4961    pub output_envelope: Option<Value>,
4962    #[serde(skip)]
4963    pub output_avro_value: Option<AvroValue>,
4964    #[serde(flatten)]
4965    pub raw: HashMap<String, Value>,
4966}
4967
4968/// Lifecycle and backlog metadata for one run-scoped Workflow Stream.
4969#[derive(Clone, Debug, Deserialize)]
4970pub struct WorkflowStreamDescription {
4971    pub stream_name: String,
4972    pub status: String,
4973    pub last_offset: i64,
4974    pub total_items: u64,
4975    pub pending_items: u64,
4976    #[serde(default)]
4977    pub opened_at: Option<String>,
4978    #[serde(default)]
4979    pub last_appended_at: Option<String>,
4980    #[serde(default)]
4981    pub closed_at: Option<String>,
4982    #[serde(default)]
4983    pub error_reason: Option<String>,
4984    #[serde(default)]
4985    pub retention_seconds: Option<u64>,
4986    #[serde(flatten)]
4987    pub raw: HashMap<String, Value>,
4988}
4989
4990impl WorkflowStreamDescription {
4991    pub fn is_terminal(&self) -> bool {
4992        matches!(self.status.as_str(), "closed" | "errored")
4993    }
4994}
4995
4996/// One item for direct or replay-safe append.
4997#[derive(Clone, Debug, Default)]
4998pub struct WorkflowStreamAppendItem {
4999    pub payload_envelope: Option<Value>,
5000    pub payload_reference: Option<String>,
5001    pub item_type: Option<String>,
5002    pub content_type: Option<String>,
5003    pub idempotency_key: Option<String>,
5004}
5005
5006impl WorkflowStreamAppendItem {
5007    /// Encode an inline payload with the SDK's fixed Avro Value envelope.
5008    pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5009        let value = AvroValue::from_serialize(&payload)?;
5010        Ok(Self {
5011            payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5012            ..Self::default()
5013        })
5014    }
5015
5016    /// Preserve an external payload URI as an opaque service-contract reference.
5017    pub fn from_reference(reference: impl Into<String>) -> Self {
5018        Self {
5019            payload_reference: Some(reference.into()),
5020            ..Self::default()
5021        }
5022    }
5023
5024    pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5025        self.item_type = Some(item_type.into());
5026        self
5027    }
5028
5029    pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5030        self.content_type = Some(content_type.into());
5031        self
5032    }
5033
5034    pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5035        self.idempotency_key = Some(idempotency_key.into());
5036        self
5037    }
5038
5039    fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5040        let mut item = serde_json::Map::new();
5041        if let Some(payload) = &self.payload_envelope {
5042            item.insert("payload".to_string(), payload.clone());
5043            item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5044        }
5045        if let Some(reference) = &self.payload_reference {
5046            item.insert("payload_reference".to_string(), json!(reference));
5047        }
5048        if let Some(item_type) = &self.item_type {
5049            item.insert("item_type".to_string(), json!(item_type));
5050        }
5051        if let Some(content_type) = &self.content_type {
5052            item.insert("content_type".to_string(), json!(content_type));
5053        }
5054        if let Some(key) = derived_idempotency_key
5055            .as_ref()
5056            .or(self.idempotency_key.as_ref())
5057        {
5058            item.insert("idempotency_key".to_string(), json!(key));
5059        }
5060        Value::Object(item)
5061    }
5062}
5063
5064/// One durable item at its stable zero-based offset.
5065#[derive(Clone, Debug)]
5066pub struct WorkflowStreamItem {
5067    pub offset: u64,
5068    pub payload: Option<Value>,
5069    pub payload_envelope: Option<Value>,
5070    pub payload_reference: Option<String>,
5071    pub payload_codec: Option<String>,
5072    pub idempotency_key: Option<String>,
5073    pub item_type: Option<String>,
5074    pub content_type: Option<String>,
5075    pub origin: Option<String>,
5076    pub origin_reference: Option<String>,
5077    pub emitted_at: Option<String>,
5078    pub raw: Value,
5079}
5080
5081/// One bounded at-least-once subscription page.
5082#[derive(Clone, Debug)]
5083pub struct WorkflowStreamPage {
5084    pub stream: WorkflowStreamDescription,
5085    pub items: Vec<WorkflowStreamItem>,
5086    pub next_offset: u64,
5087    pub terminal: bool,
5088}
5089
5090/// Durable acceptance and deduplication outcome for an append request.
5091#[derive(Clone, Debug)]
5092pub struct WorkflowStreamAppendResult {
5093    pub stream: WorkflowStreamDescription,
5094    pub accepted_offsets: Vec<u64>,
5095    pub accepted: u64,
5096    pub deduped: u64,
5097}
5098
5099#[derive(Deserialize)]
5100struct WorkflowStreamListResponse {
5101    #[serde(default)]
5102    streams: Vec<WorkflowStreamDescription>,
5103}
5104
5105#[derive(Deserialize)]
5106struct WorkflowStreamDescriptionResponse {
5107    stream: WorkflowStreamDescription,
5108}
5109
5110#[derive(Deserialize)]
5111struct WorkflowStreamPageResponse {
5112    stream: WorkflowStreamDescription,
5113    #[serde(default)]
5114    items: Vec<Value>,
5115    next_offset: u64,
5116    terminal: bool,
5117}
5118
5119#[derive(Deserialize)]
5120struct WorkflowStreamAppendResponse {
5121    stream: WorkflowStreamDescription,
5122    #[serde(default)]
5123    accepted_offsets: Vec<u64>,
5124    accepted: u64,
5125    deduped: u64,
5126}
5127
5128impl WorkflowDescription {
5129    pub fn is_completed(&self) -> bool {
5130        matches!(self.status.as_deref(), Some("completed" | "Completed"))
5131    }
5132
5133    pub fn is_terminal(&self) -> bool {
5134        matches!(
5135            self.status.as_deref(),
5136            Some(
5137                "completed"
5138                    | "Completed"
5139                    | "failed"
5140                    | "Failed"
5141                    | "cancelled"
5142                    | "Cancelled"
5143                    | "terminated"
5144                    | "Terminated"
5145                    | "timed_out"
5146                    | "TimedOut",
5147            )
5148        )
5149    }
5150
5151    fn decode_payloads(&mut self) -> Result<()> {
5152        if let Some(envelope) = &self.output_envelope {
5153            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5154            self.output = Some(value.clone().into_json()?);
5155            self.output_avro_value = Some(value);
5156        }
5157
5158        Ok(())
5159    }
5160
5161    fn raw_value(&self) -> Value {
5162        let mut data = self.raw.clone();
5163        data.insert(
5164            "workflow_id".to_string(),
5165            self.workflow_id
5166                .clone()
5167                .map(Value::String)
5168                .unwrap_or(Value::Null),
5169        );
5170        data.insert(
5171            "run_id".to_string(),
5172            self.run_id
5173                .clone()
5174                .map(Value::String)
5175                .unwrap_or(Value::Null),
5176        );
5177        data.insert(
5178            "workflow_type".to_string(),
5179            self.workflow_type
5180                .clone()
5181                .map(Value::String)
5182                .unwrap_or(Value::Null),
5183        );
5184        data.insert(
5185            "status".to_string(),
5186            self.status
5187                .clone()
5188                .map(Value::String)
5189                .unwrap_or(Value::Null),
5190        );
5191        data.insert(
5192            "closed_reason".to_string(),
5193            self.closed_reason
5194                .clone()
5195                .map(Value::String)
5196                .unwrap_or(Value::Null),
5197        );
5198        if let Some(failure) = &self.failure {
5199            data.insert("failure".to_string(), failure.clone());
5200        }
5201        if let Some(exception) = &self.exception {
5202            data.insert("exception".to_string(), exception.clone());
5203        }
5204        Value::Object(data.into_iter().collect())
5205    }
5206}
5207
5208fn workflow_terminal_outcome(
5209    description: &WorkflowDescription,
5210    workflow_id: &str,
5211    run_id: Option<&str>,
5212) -> WorkflowTerminalOutcome {
5213    let terminal_kind = description
5214        .closed_reason
5215        .as_deref()
5216        .or(description.status.as_deref())
5217        .unwrap_or("failed")
5218        .to_ascii_lowercase();
5219    let kind = match terminal_kind.as_str() {
5220        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5221        "terminated" => WorkflowTerminalKind::Terminated,
5222        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5223        _ => WorkflowTerminalKind::Failed,
5224    };
5225    let default_reason = match kind {
5226        WorkflowTerminalKind::Failed => "workflow_failed",
5227        WorkflowTerminalKind::Cancelled => "cancelled",
5228        WorkflowTerminalKind::Terminated => "terminated",
5229        WorkflowTerminalKind::TimedOut => "timed_out",
5230    };
5231    let failure = description
5232        .failure
5233        .as_ref()
5234        .filter(|value| value.is_object());
5235    let nested_failure = failure
5236        .and_then(|value| value.get("failures"))
5237        .and_then(Value::as_array)
5238        .and_then(|failures| failures.last())
5239        .or_else(|| description.failures.last());
5240    let exception = description
5241        .exception
5242        .clone()
5243        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5244        .or_else(|| {
5245            nested_failure
5246                .and_then(|value| value.get("exception_payload"))
5247                .cloned()
5248        });
5249    let string_field = |name: &str| {
5250        failure
5251            .and_then(|value| value.get(name))
5252            .and_then(Value::as_str)
5253            .or_else(|| {
5254                nested_failure
5255                    .and_then(|value| value.get(name))
5256                    .and_then(Value::as_str)
5257            })
5258            .map(str::to_string)
5259    };
5260    let exception_field = |name: &str| {
5261        exception
5262            .as_ref()
5263            .and_then(|value| value.get(name))
5264            .and_then(Value::as_str)
5265            .map(str::to_string)
5266    };
5267    let message = description
5268        .error
5269        .clone()
5270        .or_else(|| string_field("message"))
5271        .or_else(|| exception_field("message"));
5272    let reason = description
5273        .raw
5274        .get("reason")
5275        .and_then(Value::as_str)
5276        .map(str::to_string)
5277        .or_else(|| {
5278            failure
5279                .and_then(|value| value.get("reason"))
5280                .and_then(Value::as_str)
5281                .map(str::to_string)
5282        })
5283        .or_else(|| description.closed_reason.clone())
5284        .unwrap_or_else(|| default_reason.to_string());
5285    let failure_id = string_field("failure_id").or_else(|| {
5286        nested_failure
5287            .and_then(|value| value.get("id"))
5288            .and_then(Value::as_str)
5289            .map(str::to_string)
5290    });
5291
5292    WorkflowTerminalOutcome {
5293        kind,
5294        workflow_id: description
5295            .workflow_id
5296            .clone()
5297            .unwrap_or_else(|| workflow_id.to_string()),
5298        run_id: description
5299            .run_id
5300            .clone()
5301            .or_else(|| run_id.map(str::to_string)),
5302        reason,
5303        failure_category: string_field("failure_category")
5304            .or_else(|| Some(default_reason.to_string())),
5305        failure_id,
5306        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5307        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5308        non_retryable: failure
5309            .and_then(|value| value.get("non_retryable"))
5310            .and_then(Value::as_bool)
5311            .or_else(|| {
5312                nested_failure
5313                    .and_then(|value| value.get("non_retryable"))
5314                    .and_then(Value::as_bool)
5315            }),
5316        message,
5317        exception,
5318        raw: description.raw_value(),
5319    }
5320}
5321
5322#[derive(Clone, Debug, Deserialize)]
5323pub struct RegisterWorkerResponse {
5324    pub worker_id: String,
5325    pub registered: bool,
5326    #[serde(default)]
5327    pub heartbeat_interval_seconds: Option<u64>,
5328    #[serde(default)]
5329    pub protocol_version: Option<String>,
5330    #[serde(default)]
5331    pub server_capabilities: Option<Value>,
5332}
5333
5334/// Result of gracefully removing a worker-plane registration.
5335#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5336pub struct WorkerDeregistrationEnvelope {
5337    pub worker_id: String,
5338    pub outcome: String,
5339    pub recovered_workflow_task_count: u64,
5340}
5341
5342#[derive(Clone, Debug, Deserialize)]
5343pub struct PollWorkflowTaskResponse {
5344    #[serde(default)]
5345    pub task: Option<WorkflowTask>,
5346    #[serde(default)]
5347    pub poll_status: Option<String>,
5348    #[serde(default)]
5349    pub reason: Option<String>,
5350    #[serde(default)]
5351    pub protocol_version: Option<String>,
5352    #[serde(default)]
5353    pub server_capabilities: Option<Value>,
5354}
5355
5356impl PollWorkflowTaskResponse {
5357    /// Classify this response without parsing server display text.
5358    pub fn outcome(&self) -> WorkerPollOutcome {
5359        worker_poll_outcome(
5360            self.task.is_some(),
5361            self.poll_status.as_deref(),
5362            self.reason.as_deref(),
5363        )
5364    }
5365}
5366
5367fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5368    let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5369        return false;
5370    };
5371    let supported = capabilities
5372        .get("workflow_memo_updates")
5373        .and_then(Value::as_object)
5374        .and_then(|memo| memo.get("supported"))
5375        .and_then(Value::as_bool)
5376        == Some(true);
5377    let command_advertised = capabilities
5378        .get("supported_workflow_task_commands")
5379        .and_then(Value::as_array)
5380        .is_some_and(|commands| {
5381            commands
5382                .iter()
5383                .any(|command| command.as_str() == Some("upsert_memo"))
5384        });
5385    supported && command_advertised
5386}
5387
5388fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5389    commands
5390        .iter()
5391        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5392}
5393
5394#[derive(Clone, Debug, Deserialize)]
5395pub struct PollActivityTaskResponse {
5396    #[serde(default)]
5397    pub task: Option<ActivityTask>,
5398    #[serde(default)]
5399    pub poll_status: Option<String>,
5400    #[serde(default)]
5401    pub reason: Option<String>,
5402}
5403
5404impl PollActivityTaskResponse {
5405    /// Classify this response without parsing server display text.
5406    pub fn outcome(&self) -> WorkerPollOutcome {
5407        worker_poll_outcome(
5408            self.task.is_some(),
5409            self.poll_status.as_deref(),
5410            self.reason.as_deref(),
5411        )
5412    }
5413}
5414
5415#[derive(Clone, Debug, Deserialize)]
5416pub struct PollQueryTaskResponse {
5417    #[serde(default)]
5418    pub task: Option<QueryTask>,
5419    #[serde(default)]
5420    pub poll_status: Option<String>,
5421    #[serde(default)]
5422    pub reason: Option<String>,
5423}
5424
5425impl PollQueryTaskResponse {
5426    /// Classify this response without parsing server display text.
5427    pub fn outcome(&self) -> WorkerPollOutcome {
5428        worker_poll_outcome(
5429            self.task.is_some(),
5430            self.poll_status.as_deref(),
5431            self.reason.as_deref(),
5432        )
5433    }
5434}
5435
5436/// Stable classification for worker poll responses.
5437#[derive(Clone, Debug, PartialEq, Eq)]
5438pub enum WorkerPollOutcome {
5439    /// A task was leased and is available on the response.
5440    Task,
5441    /// No task was leased, but the worker should continue polling.
5442    Idle {
5443        poll_status: Option<String>,
5444        reason: Option<String>,
5445    },
5446    /// The server asked this worker to stop claiming new work.
5447    Stop {
5448        poll_status: Option<String>,
5449        reason: Option<String>,
5450    },
5451}
5452
5453impl WorkerPollOutcome {
5454    pub fn should_stop(&self) -> bool {
5455        matches!(self, Self::Stop { .. })
5456    }
5457}
5458
5459fn worker_poll_outcome(
5460    has_task: bool,
5461    poll_status: Option<&str>,
5462    reason: Option<&str>,
5463) -> WorkerPollOutcome {
5464    if worker_poll_is_stop(poll_status, reason) {
5465        return WorkerPollOutcome::Stop {
5466            poll_status: poll_status.map(str::to_string),
5467            reason: reason.map(str::to_string),
5468        };
5469    }
5470
5471    if has_task {
5472        WorkerPollOutcome::Task
5473    } else {
5474        WorkerPollOutcome::Idle {
5475            poll_status: poll_status.map(str::to_string),
5476            reason: reason.map(str::to_string),
5477        }
5478    }
5479}
5480
5481/// An ephemeral server-routed query task.
5482#[derive(Clone, Debug, Deserialize)]
5483pub struct QueryTask {
5484    pub query_task_id: String,
5485    #[serde(default = "default_workflow_task_attempt")]
5486    pub query_task_attempt: u64,
5487    #[serde(default)]
5488    pub lease_owner: Option<String>,
5489    #[serde(default)]
5490    pub workflow_id: Option<String>,
5491    #[serde(default)]
5492    pub run_id: Option<String>,
5493    pub workflow_type: String,
5494    pub query_name: String,
5495    #[serde(
5496        default = "missing_task_payload_codec",
5497        deserialize_with = "deserialize_task_payload_codec"
5498    )]
5499    pub payload_codec: String,
5500    #[serde(default)]
5501    pub workflow_arguments: Option<Value>,
5502    #[serde(default)]
5503    pub query_arguments: Option<Value>,
5504    #[serde(default)]
5505    pub history_events: Vec<HistoryEvent>,
5506    #[serde(default)]
5507    pub history_export: Option<Value>,
5508    #[serde(default)]
5509    pub run_status: Option<String>,
5510}
5511
5512#[derive(Clone, Debug, Deserialize)]
5513pub struct WorkflowTask {
5514    pub task_id: String,
5515    #[serde(default)]
5516    pub workflow_command_id: Option<String>,
5517    #[serde(default)]
5518    pub workflow_id: Option<String>,
5519    #[serde(default)]
5520    pub run_id: Option<String>,
5521    pub workflow_type: String,
5522    #[serde(default)]
5523    pub cancel_requested: bool,
5524    #[serde(
5525        default = "missing_task_payload_codec",
5526        deserialize_with = "deserialize_task_payload_codec"
5527    )]
5528    pub payload_codec: String,
5529    #[serde(default)]
5530    pub arguments: Option<Value>,
5531    #[serde(default)]
5532    pub history_events: Vec<HistoryEvent>,
5533    #[serde(default)]
5534    pub total_history_events: Option<u64>,
5535    #[serde(default)]
5536    pub history_size_bytes: Option<u64>,
5537    #[serde(default)]
5538    pub continue_as_new_recommended: Option<bool>,
5539    #[serde(default)]
5540    pub history_budget_pressure: Option<String>,
5541    #[serde(default)]
5542    pub next_history_page_token: Option<String>,
5543    #[serde(default = "default_workflow_task_attempt")]
5544    pub workflow_task_attempt: u64,
5545    #[serde(default)]
5546    pub workflow_signal_id: Option<String>,
5547    #[serde(default)]
5548    pub signal_name: Option<String>,
5549    #[serde(default)]
5550    pub signal_arguments: Option<Value>,
5551    #[serde(default)]
5552    pub workflow_update_id: Option<String>,
5553    #[serde(default)]
5554    pub update_name: Option<String>,
5555    #[serde(default)]
5556    pub lease_owner: Option<String>,
5557}
5558
5559impl WorkflowTask {
5560    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5561        self.history_events.extend(page.history_events);
5562
5563        if page.total_history_events.is_some() {
5564            self.total_history_events = page.total_history_events;
5565        }
5566
5567        self.next_history_page_token = page
5568            .next_history_page_token
5569            .filter(|token| !token.is_empty());
5570    }
5571}
5572
5573#[derive(Clone, Debug, Deserialize)]
5574struct WorkflowTaskHistoryPage {
5575    #[serde(default)]
5576    history_events: Vec<HistoryEvent>,
5577    #[serde(default)]
5578    total_history_events: Option<u64>,
5579    #[serde(default)]
5580    next_history_page_token: Option<String>,
5581}
5582
5583#[derive(Clone, Debug, Deserialize)]
5584pub struct ActivityTask {
5585    pub task_id: String,
5586    #[serde(default)]
5587    pub activity_attempt_id: Option<String>,
5588    #[serde(default)]
5589    pub attempt_id: Option<String>,
5590    pub activity_type: String,
5591    #[serde(
5592        default = "missing_task_payload_codec",
5593        deserialize_with = "deserialize_task_payload_codec"
5594    )]
5595    pub payload_codec: String,
5596    #[serde(default)]
5597    pub arguments: Option<Value>,
5598    #[serde(default = "default_attempt_number")]
5599    pub attempt_number: u64,
5600    #[serde(default)]
5601    pub lease_owner: Option<String>,
5602}
5603
5604#[derive(Clone, Debug, Deserialize)]
5605pub struct HistoryEvent {
5606    #[serde(alias = "type")]
5607    pub event_type: String,
5608    #[serde(default)]
5609    pub payload: Value,
5610    #[serde(flatten)]
5611    pub raw: HashMap<String, Value>,
5612}
5613
5614/// One decoded signal in the committed workflow-history snapshot.
5615#[derive(Clone, Debug, PartialEq)]
5616pub struct QuerySignal {
5617    pub id: Option<String>,
5618    pub name: String,
5619    pub arguments: Vec<Value>,
5620    avro_arguments: Vec<AvroValue>,
5621    pub workflow_sequence: Option<u64>,
5622}
5623
5624impl QuerySignal {
5625    /// Lossless fixed Avro Value arguments for this committed signal.
5626    pub fn arguments_avro_value(&self) -> &[AvroValue] {
5627        &self.avro_arguments
5628    }
5629}
5630
5631/// Immutable state supplied to a registered query handler.
5632///
5633/// This context intentionally exposes no activity, signal-wait, or command
5634/// APIs. Query handlers inspect committed history and return a value; query
5635/// completion does not append an event or advance deterministic execution.
5636#[derive(Clone, Debug)]
5637pub struct QueryContext {
5638    pub workflow_id: Option<String>,
5639    pub run_id: Option<String>,
5640    pub workflow_type: String,
5641    pub run_status: Option<String>,
5642    workflow_input: Value,
5643    workflow_input_avro_value: AvroValue,
5644    history_events: Arc<Vec<HistoryEvent>>,
5645    signal_events: Arc<Vec<QuerySignal>>,
5646}
5647
5648impl QueryContext {
5649    /// The normalized argument list used to start the workflow.
5650    pub fn workflow_input(&self) -> &Value {
5651        &self.workflow_input
5652    }
5653
5654    /// The lossless fixed Avro Value argument list used to start the workflow.
5655    pub fn workflow_input_avro_value(&self) -> &AvroValue {
5656        &self.workflow_input_avro_value
5657    }
5658
5659    /// The immutable committed history used for this query snapshot.
5660    pub fn history_events(&self) -> &[HistoryEvent] {
5661        self.history_events.as_slice()
5662    }
5663
5664    /// All decoded signals in committed workflow order.
5665    pub fn signal_events(&self) -> &[QuerySignal] {
5666        self.signal_events.as_slice()
5667    }
5668
5669    /// Decoded argument lists for each committed signal with `signal_name`.
5670    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5671        self.signal_events
5672            .iter()
5673            .filter(|signal| signal.name == signal_name)
5674            .map(|signal| signal.arguments.clone())
5675            .collect()
5676    }
5677
5678    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
5679    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5680        self.signal_events
5681            .iter()
5682            .filter(|signal| signal.name == signal_name)
5683            .map(|signal| signal.avro_arguments.clone())
5684            .collect()
5685    }
5686}
5687
5688#[derive(Clone, Debug, Deserialize)]
5689pub struct ActivityHeartbeatResponse {
5690    #[serde(default)]
5691    pub cancel_requested: bool,
5692    #[serde(default)]
5693    pub heartbeat_recorded: bool,
5694    #[serde(default)]
5695    pub can_continue: Option<bool>,
5696    #[serde(default)]
5697    pub reason: Option<String>,
5698    #[serde(default)]
5699    pub run_closed_reason: Option<String>,
5700    #[serde(default)]
5701    pub run_closed_at: Option<String>,
5702    #[serde(default)]
5703    pub lease_expires_at: Option<String>,
5704    #[serde(default)]
5705    pub last_heartbeat_at: Option<String>,
5706}
5707
5708impl ActivityHeartbeatResponse {
5709    /// Whether the activity should stop instead of attempting completion.
5710    pub fn should_stop(&self) -> bool {
5711        self.cancel_requested || self.can_continue == Some(false)
5712    }
5713}
5714
5715fn missing_task_payload_codec() -> String {
5716    MISSING_TASK_PAYLOAD_CODEC.to_string()
5717}
5718
5719fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5720where
5721    D: Deserializer<'de>,
5722{
5723    Ok(match Value::deserialize(deserializer)? {
5724        Value::String(codec) => codec,
5725        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5726        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5727    })
5728}
5729
5730fn default_workflow_task_attempt() -> u64 {
5731    1
5732}
5733
5734fn default_attempt_number() -> u64 {
5735    1
5736}
5737
5738type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5739type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5740type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5741type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5742type ReplayedWorkflowHandler =
5743    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5744type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5745type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5746type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5747type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5748type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5749type ReplayedQueryHandler = Arc<
5750    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5751        + Send
5752        + Sync,
5753>;
5754type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5755
5756struct ReplayedWorkflowInvocation {
5757    future: WorkflowFuture,
5758    snapshot: WorkflowStateSnapshot,
5759}
5760
5761#[derive(Clone)]
5762struct RegisteredWorkflow {
5763    execute: WorkflowHandler,
5764    replay: Option<ReplayedWorkflowHandler>,
5765    state_type: Option<TypeId>,
5766}
5767
5768#[derive(Debug)]
5769struct WorkflowTaskDecision {
5770    commands: Vec<Value>,
5771    message_stream_cursors: Vec<Value>,
5772    message_stream_waits: Vec<Value>,
5773}
5774
5775impl WorkflowTaskDecision {
5776    fn without_message_streams(commands: Vec<Value>) -> Self {
5777        Self {
5778            commands,
5779            message_stream_cursors: Vec::new(),
5780            message_stream_waits: Vec::new(),
5781        }
5782    }
5783}
5784
5785#[derive(Clone)]
5786enum RegisteredQuery {
5787    Snapshot(QueryHandler),
5788    Replayed {
5789        state_type: TypeId,
5790        handler: ReplayedQueryHandler,
5791    },
5792}
5793
5794#[derive(Clone, Debug)]
5795pub struct WorkerHeartbeatObservation {
5796    pub worker_id: String,
5797    pub task_queue: String,
5798    pub acknowledged_at_unix_millis: u64,
5799    pub acknowledgement: Value,
5800}
5801
5802/// Bounded retry policy for worker poll acquisition and worker heartbeats.
5803///
5804/// Expected empty long polls and explicit Server capacity backpressure are
5805/// normal worker states. Capacity backpressure honors the advertised retry
5806/// delay without consuming this retry budget. Other transport failures,
5807/// retryable HTTP 408/429 responses, and server errors are retried with capped
5808/// exponential backoff. Authentication, protocol, codec, and handler failures
5809/// are never retried by the worker.
5810/// Explicit storage admission pauses also preserve the already-serialized worker
5811/// request without consuming this budget or rerunning its handler. The delay is
5812/// capped by `max_backoff`, and shutdown interrupts storage waits.
5813#[derive(Clone, Copy, Debug)]
5814pub struct WorkerRetryPolicy {
5815    /// Number of retries after the initial request fails.
5816    pub max_retries: usize,
5817    /// Delay before the first retry.
5818    pub initial_backoff: Duration,
5819    /// Maximum delay between retries.
5820    pub max_backoff: Duration,
5821}
5822
5823impl Default for WorkerRetryPolicy {
5824    fn default() -> Self {
5825        Self {
5826            max_retries: 5,
5827            initial_backoff: Duration::from_millis(100),
5828            max_backoff: Duration::from_secs(5),
5829        }
5830    }
5831}
5832
5833#[derive(Clone, Debug)]
5834struct WorkerStorageAdmission {
5835    policy: WorkerRetryPolicy,
5836    stop: Arc<AtomicBool>,
5837}
5838
5839struct StopWorkerOnDrop(Arc<AtomicBool>);
5840
5841impl Drop for StopWorkerOnDrop {
5842    fn drop(&mut self) {
5843        self.0.store(true, Ordering::SeqCst);
5844    }
5845}
5846
5847async fn wait_for_worker_stop(stop: &AtomicBool) {
5848    while !stop.load(Ordering::SeqCst) {
5849        tokio::time::sleep(Duration::from_millis(100)).await;
5850    }
5851}
5852
5853#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5854enum ManagedPollOutcome {
5855    Idle,
5856    Handled,
5857    Stop,
5858}
5859
5860#[derive(Clone)]
5861pub struct Worker {
5862    client: Client,
5863    worker_id: String,
5864    task_queue: String,
5865    workflows: HashMap<String, RegisteredWorkflow>,
5866    activities: HashMap<String, ActivityHandler>,
5867    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5868    updates: HashMap<String, HashMap<String, UpdateHandler>>,
5869    max_concurrent_workflow_tasks: usize,
5870    max_concurrent_activity_tasks: usize,
5871    poll_timeout: Duration,
5872    heartbeat_interval: Duration,
5873    retry_policy: WorkerRetryPolicy,
5874    heartbeat_observer: Option<WorkerHeartbeatObserver>,
5875}
5876
5877impl Worker {
5878    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5879        Self {
5880            client,
5881            worker_id: default_worker_id(),
5882            task_queue: task_queue.into(),
5883            workflows: HashMap::new(),
5884            activities: HashMap::new(),
5885            queries: HashMap::new(),
5886            updates: HashMap::new(),
5887            max_concurrent_workflow_tasks: 10,
5888            max_concurrent_activity_tasks: 10,
5889            poll_timeout: Duration::from_secs(30),
5890            heartbeat_interval: Duration::from_secs(60),
5891            retry_policy: WorkerRetryPolicy::default(),
5892            heartbeat_observer: None,
5893        }
5894    }
5895
5896    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5897        self.worker_id = worker_id.into();
5898        self
5899    }
5900
5901    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5902        self.poll_timeout = timeout;
5903        self
5904    }
5905
5906    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5907        self.heartbeat_interval = interval;
5908        self
5909    }
5910
5911    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
5912    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5913        self.retry_policy = policy;
5914        self
5915    }
5916
5917    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5918    where
5919        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5920    {
5921        self.heartbeat_observer = Some(Arc::new(observer));
5922        self
5923    }
5924
5925    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5926        self.max_concurrent_workflow_tasks = count.max(1);
5927        self
5928    }
5929
5930    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5931        self.max_concurrent_activity_tasks = count.max(1);
5932        self
5933    }
5934
5935    /// Register a workflow handler.
5936    ///
5937    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
5938    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
5939    /// while acquiring or decoding a worker task remain worker-operation
5940    /// failures and do not get converted into workflow outcomes.
5941    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5942    where
5943        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5944        Fut: Future<Output = Result<Value>> + Send + 'static,
5945    {
5946        let handler = Arc::new(handler);
5947        self.workflows.insert(
5948            workflow_type.into(),
5949            RegisteredWorkflow {
5950                execute: Arc::new(move |ctx, input| {
5951                    let handler = Arc::clone(&handler);
5952                    Box::pin(async move {
5953                        let result = handler(ctx, input.into_json()?).await?;
5954                        AvroValue::from_serialize(&result)
5955                    })
5956                }),
5957                replay: None,
5958                state_type: None,
5959            },
5960        );
5961    }
5962
5963    /// Register a workflow with one Serde request value and a Serde result.
5964    ///
5965    /// This is an ergonomic adapter over the same fixed Avro Value protocol as
5966    /// [`Worker::register_workflow_avro_value`]. It does not create or publish a
5967    /// workflow-specific schema. A task must contain zero arguments for a unit
5968    /// request or exactly one argument for every other request type.
5969    ///
5970    /// See the runnable
5971    /// [`hello_world` example](https://github.com/durable-workflow/sdk-rust/blob/main/examples/hello_world.rs)
5972    /// for typed workflow and activity contracts with retry and timeout policy.
5973    pub fn register_typed_workflow<I, O, F, Fut>(
5974        &mut self,
5975        workflow_type: impl Into<String>,
5976        handler: F,
5977    ) where
5978        I: DeserializeOwned + Send + 'static,
5979        O: Serialize + Send + 'static,
5980        F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
5981        Fut: Future<Output = Result<O>> + Send + 'static,
5982    {
5983        let workflow_type = workflow_type.into();
5984        let handler_name = workflow_type.clone();
5985        let handler = Arc::new(handler);
5986        self.workflows.insert(
5987            workflow_type,
5988            RegisteredWorkflow {
5989                execute: Arc::new(move |ctx, input| {
5990                    let handler = Arc::clone(&handler);
5991                    let handler_name = handler_name.clone();
5992                    Box::pin(async move {
5993                        let input =
5994                            decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5995                        let result = handler(ctx, input).await?;
5996                        encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5997                    })
5998                }),
5999                replay: None,
6000                state_type: None,
6001            },
6002        );
6003    }
6004
6005    /// Register a workflow on the lossless fixed Avro Value surface.
6006    pub fn register_workflow_avro_value<F, Fut>(
6007        &mut self,
6008        workflow_type: impl Into<String>,
6009        handler: F,
6010    ) where
6011        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6012        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6013    {
6014        self.workflows.insert(
6015            workflow_type.into(),
6016            RegisteredWorkflow {
6017                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6018                replay: None,
6019                state_type: None,
6020            },
6021        );
6022    }
6023
6024    /// Register a workflow whose typed instance state can be reconstructed for queries.
6025    ///
6026    /// `state_factory` creates a fresh instance for every normal workflow task and
6027    /// query replay. The workflow handler is the single source of truth for state
6028    /// transitions: it updates [`WorkflowInstance`] after activities and signals
6029    /// resolve. Query replay runs this same handler over committed history and
6030    /// discards any commands it would emit.
6031    pub fn register_replayed_workflow<S, Factory, F, Fut>(
6032        &mut self,
6033        workflow_type: impl Into<String>,
6034        state_factory: Factory,
6035        handler: F,
6036    ) where
6037        S: Clone + Send + Sync + 'static,
6038        Factory: Fn() -> S + Send + Sync + 'static,
6039        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6040        Fut: Future<Output = Result<Value>> + Send + 'static,
6041    {
6042        let state_factory = Arc::new(state_factory);
6043        let handler = Arc::new(handler);
6044
6045        let execute_factory = Arc::clone(&state_factory);
6046        let execute_handler = Arc::clone(&handler);
6047        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6048            let state = WorkflowInstance::new(execute_factory());
6049            let handler = Arc::clone(&execute_handler);
6050            Box::pin(async move {
6051                let result = handler(ctx, input.into_json()?, state).await?;
6052                AvroValue::from_serialize(&result)
6053            }) as WorkflowFuture
6054        });
6055
6056        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6057            let state = WorkflowInstance::new(state_factory());
6058            let snapshot_state = state.clone();
6059            let snapshot: WorkflowStateSnapshot =
6060                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6061            let replay_handler = Arc::clone(&handler);
6062            let future = async move {
6063                let result = replay_handler(ctx, input.into_json()?, state).await?;
6064                AvroValue::from_serialize(&result)
6065            };
6066            ReplayedWorkflowInvocation {
6067                future: Box::pin(future),
6068                snapshot,
6069            }
6070        });
6071
6072        self.workflows.insert(
6073            workflow_type.into(),
6074            RegisteredWorkflow {
6075                execute,
6076                replay: Some(replay),
6077                state_type: Some(TypeId::of::<S>()),
6078            },
6079        );
6080    }
6081
6082    /// Register a replayable workflow with one Serde request value and result.
6083    ///
6084    /// Normal task execution and instance-state query replay both decode and
6085    /// encode through the fixed Avro Value codec. The state factory and handler
6086    /// otherwise follow [`Worker::register_replayed_workflow`].
6087    pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6088        &mut self,
6089        workflow_type: impl Into<String>,
6090        state_factory: Factory,
6091        handler: F,
6092    ) where
6093        I: DeserializeOwned + Send + 'static,
6094        O: Serialize + Send + 'static,
6095        S: Clone + Send + Sync + 'static,
6096        Factory: Fn() -> S + Send + Sync + 'static,
6097        F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6098        Fut: Future<Output = Result<O>> + Send + 'static,
6099    {
6100        let workflow_type = workflow_type.into();
6101        let state_factory = Arc::new(state_factory);
6102        let handler = Arc::new(handler);
6103
6104        let execute_name = workflow_type.clone();
6105        let execute_factory = Arc::clone(&state_factory);
6106        let execute_handler = Arc::clone(&handler);
6107        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6108            let state = WorkflowInstance::new(execute_factory());
6109            let handler = Arc::clone(&execute_handler);
6110            let handler_name = execute_name.clone();
6111            Box::pin(async move {
6112                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6113                let result = handler(ctx, input, state).await?;
6114                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6115            }) as WorkflowFuture
6116        });
6117
6118        let replay_name = workflow_type.clone();
6119        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6120            let state = WorkflowInstance::new(state_factory());
6121            let snapshot_state = state.clone();
6122            let snapshot: WorkflowStateSnapshot =
6123                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6124            let handler = Arc::clone(&handler);
6125            let handler_name = replay_name.clone();
6126            let future = async move {
6127                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6128                let result = handler(ctx, input, state).await?;
6129                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6130            };
6131            ReplayedWorkflowInvocation {
6132                future: Box::pin(future),
6133                snapshot,
6134            }
6135        });
6136
6137        self.workflows.insert(
6138            workflow_type,
6139            RegisteredWorkflow {
6140                execute,
6141                replay: Some(replay),
6142                state_type: Some(TypeId::of::<S>()),
6143            },
6144        );
6145    }
6146
6147    /// Register a replayable workflow on the lossless fixed Avro Value surface.
6148    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6149        &mut self,
6150        workflow_type: impl Into<String>,
6151        state_factory: Factory,
6152        handler: F,
6153    ) where
6154        S: Clone + Send + Sync + 'static,
6155        Factory: Fn() -> S + Send + Sync + 'static,
6156        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6157        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6158    {
6159        let state_factory = Arc::new(state_factory);
6160        let handler = Arc::new(handler);
6161
6162        let execute_factory = Arc::clone(&state_factory);
6163        let execute_handler = Arc::clone(&handler);
6164        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6165            let state = WorkflowInstance::new(execute_factory());
6166            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6167        });
6168
6169        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6170            let state = WorkflowInstance::new(state_factory());
6171            let snapshot_state = state.clone();
6172            let snapshot: WorkflowStateSnapshot =
6173                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6174            ReplayedWorkflowInvocation {
6175                future: Box::pin(handler(ctx, input, state)),
6176                snapshot,
6177            }
6178        });
6179
6180        self.workflows.insert(
6181            workflow_type.into(),
6182            RegisteredWorkflow {
6183                execute,
6184                replay: Some(replay),
6185                state_type: Some(TypeId::of::<S>()),
6186            },
6187        );
6188    }
6189
6190    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6191    where
6192        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6193        Fut: Future<Output = Result<Value>> + Send + 'static,
6194    {
6195        let handler = Arc::new(handler);
6196        self.activities.insert(
6197            activity_type.into(),
6198            Arc::new(move |ctx, args| {
6199                let handler = Arc::clone(&handler);
6200                Box::pin(async move {
6201                    let result = handler(ctx, args.into_json()?).await?;
6202                    AvroValue::from_serialize(&result)
6203                })
6204            }),
6205        );
6206    }
6207
6208    /// Register an activity with one Serde request value and a Serde result.
6209    ///
6210    /// Inputs and results use the platform's fixed Avro Value schema. Shape
6211    /// mismatches and unsupported Serde values return [`Error::HandlerType`]
6212    /// with the activity name and Rust type.
6213    pub fn register_typed_activity<I, O, F, Fut>(
6214        &mut self,
6215        activity_type: impl Into<String>,
6216        handler: F,
6217    ) where
6218        I: DeserializeOwned + Send + 'static,
6219        O: Serialize + Send + 'static,
6220        F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6221        Fut: Future<Output = Result<O>> + Send + 'static,
6222    {
6223        let activity_type = activity_type.into();
6224        let handler_name = activity_type.clone();
6225        let handler = Arc::new(handler);
6226        self.activities.insert(
6227            activity_type,
6228            Arc::new(move |ctx, input| {
6229                let handler = Arc::clone(&handler);
6230                let handler_name = handler_name.clone();
6231                Box::pin(async move {
6232                    let input =
6233                        decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6234                    let result = handler(ctx, input).await?;
6235                    encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6236                })
6237            }),
6238        );
6239    }
6240
6241    /// Register an activity on the lossless fixed Avro Value surface.
6242    pub fn register_activity_avro_value<F, Fut>(
6243        &mut self,
6244        activity_type: impl Into<String>,
6245        handler: F,
6246    ) where
6247        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6248        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6249    {
6250        self.activities.insert(
6251            activity_type.into(),
6252            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6253        );
6254    }
6255
6256    /// Register a named, read-only query handler for a workflow type.
6257    ///
6258    /// The workflow type must also be registered with [`Worker::register_workflow`]
6259    /// before the worker runs. The handler receives only an immutable committed
6260    /// state snapshot and normalized query arguments.
6261    pub fn register_query<F, Fut>(
6262        &mut self,
6263        workflow_type: impl Into<String>,
6264        query_name: impl Into<String>,
6265        handler: F,
6266    ) where
6267        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6268        Fut: Future<Output = Result<Value>> + Send + 'static,
6269    {
6270        let handler = Arc::new(handler);
6271        self.queries
6272            .entry(workflow_type.into())
6273            .or_default()
6274            .insert(
6275                query_name.into(),
6276                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6277                    let handler = Arc::clone(&handler);
6278                    Box::pin(async move {
6279                        let result = handler(ctx, args.into_json()?).await?;
6280                        AvroValue::from_serialize(&result)
6281                    })
6282                })),
6283            );
6284    }
6285
6286    /// Register a query handler on the lossless fixed Avro Value surface.
6287    pub fn register_query_avro_value<F, Fut>(
6288        &mut self,
6289        workflow_type: impl Into<String>,
6290        query_name: impl Into<String>,
6291        handler: F,
6292    ) where
6293        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6294        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6295    {
6296        self.queries
6297            .entry(workflow_type.into())
6298            .or_default()
6299            .insert(
6300                query_name.into(),
6301                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6302            );
6303    }
6304
6305    /// Register a named query against deterministically replayed instance state.
6306    ///
6307    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
6308    /// same state type `S`. The handler receives an immutable, detached state
6309    /// clone, so successful and failed queries cannot affect workflow execution
6310    /// or the state reconstructed by a later query.
6311    pub fn register_replayed_query<S, F, Fut>(
6312        &mut self,
6313        workflow_type: impl Into<String>,
6314        query_name: impl Into<String>,
6315        handler: F,
6316    ) where
6317        S: Clone + Send + Sync + 'static,
6318        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6319        Fut: Future<Output = Result<Value>> + Send + 'static,
6320    {
6321        let handler = Arc::new(handler);
6322        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6323            let state = state.downcast::<S>().map_err(|_| {
6324                "registered query state type does not match the replayed workflow state".to_string()
6325            })?;
6326            let handler = Arc::clone(&handler);
6327            Ok(Box::pin(async move {
6328                let result = handler(ctx, state, args.into_json()?).await?;
6329                AvroValue::from_serialize(&result)
6330            }))
6331        });
6332
6333        self.queries
6334            .entry(workflow_type.into())
6335            .or_default()
6336            .insert(
6337                query_name.into(),
6338                RegisteredQuery::Replayed {
6339                    state_type: TypeId::of::<S>(),
6340                    handler: erased_handler,
6341                },
6342            );
6343    }
6344
6345    /// Register a replayed-state query on the lossless fixed Avro Value surface.
6346    pub fn register_replayed_query_avro_value<S, F, Fut>(
6347        &mut self,
6348        workflow_type: impl Into<String>,
6349        query_name: impl Into<String>,
6350        handler: F,
6351    ) where
6352        S: Clone + Send + Sync + 'static,
6353        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6354        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6355    {
6356        let handler = Arc::new(handler);
6357        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6358            let state = state.downcast::<S>().map_err(|_| {
6359                "registered query state type does not match the replayed workflow state".to_string()
6360            })?;
6361            Ok(Box::pin(handler(ctx, state, args)))
6362        });
6363
6364        self.queries
6365            .entry(workflow_type.into())
6366            .or_default()
6367            .insert(
6368                query_name.into(),
6369                RegisteredQuery::Replayed {
6370                    state_type: TypeId::of::<S>(),
6371                    handler: erased_handler,
6372                },
6373            );
6374    }
6375
6376    /// Register a synchronous workflow update handler.
6377    pub fn register_update<F, Fut>(
6378        &mut self,
6379        workflow_type: impl Into<String>,
6380        update_name: impl Into<String>,
6381        handler: F,
6382    ) where
6383        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6384        Fut: Future<Output = Result<Value>> + Send + 'static,
6385    {
6386        let handler = Arc::new(handler);
6387        self.updates
6388            .entry(workflow_type.into())
6389            .or_default()
6390            .insert(
6391                update_name.into(),
6392                Arc::new(move |ctx, args| {
6393                    let handler = Arc::clone(&handler);
6394                    Box::pin(async move {
6395                        let result = handler(ctx, args.into_json()?).await?;
6396                        AvroValue::from_serialize(&result)
6397                    })
6398                }),
6399            );
6400    }
6401
6402    /// Register an update handler on the lossless fixed Avro Value surface.
6403    pub fn register_update_avro_value<F, Fut>(
6404        &mut self,
6405        workflow_type: impl Into<String>,
6406        update_name: impl Into<String>,
6407        handler: F,
6408    ) where
6409        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6410        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6411    {
6412        self.updates
6413            .entry(workflow_type.into())
6414            .or_default()
6415            .insert(
6416                update_name.into(),
6417                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6418            );
6419    }
6420
6421    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6422        let mut command_contracts = serde_json::Map::new();
6423        for workflow_type in self.workflows.keys() {
6424            let mut queries = self
6425                .queries
6426                .get(workflow_type)
6427                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6428                .unwrap_or_default();
6429            queries.sort();
6430            let mut updates = self
6431                .updates
6432                .get(workflow_type)
6433                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6434                .unwrap_or_default();
6435            updates.sort();
6436            command_contracts.insert(
6437                workflow_type.clone(),
6438                json!({
6439                    "queries": queries,
6440                    "query_contracts": [],
6441                    "signals": [],
6442                    "signal_contracts": [],
6443                    "updates": updates,
6444                    "update_contracts": [],
6445                    "update_validators": [],
6446                }),
6447            );
6448        }
6449
6450        self.client
6451            .register_worker_with_command_contracts(
6452                &self.worker_id,
6453                &self.task_queue,
6454                self.workflows.keys().cloned().collect(),
6455                self.activities.keys().cloned().collect(),
6456                self.max_concurrent_workflow_tasks,
6457                self.max_concurrent_activity_tasks,
6458                [
6459                    Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6460                    Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6461                    Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6462                    Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6463                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6464                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6465                    worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6466                        .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6467                ]
6468                .into_iter()
6469                .flatten()
6470                .collect(),
6471                Value::Object(command_contracts),
6472            )
6473            .await
6474    }
6475
6476    /// Run until shutdown or a terminal worker error occurs.
6477    ///
6478    /// Empty long-poll expirations do not stop the worker. Retryable poll and
6479    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
6480    /// authentication, protocol, and other non-retryable failures are returned.
6481    pub async fn run(&self) -> Result<()> {
6482        self.run_until(std::future::pending::<()>()).await
6483    }
6484
6485    /// Run until `shutdown` resolves or a terminal worker error occurs.
6486    ///
6487    /// This has the same liveness and terminal-error contract as [`Worker::run`].
6488    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6489    where
6490        F: Future<Output = ()>,
6491    {
6492        let stop = Arc::new(AtomicBool::new(false));
6493        let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6494        let worker = self.with_storage_admission(Arc::clone(&stop));
6495        let run = worker.run_with_storage_admission(Arc::clone(&stop));
6496        tokio::pin!(run);
6497        tokio::pin!(shutdown);
6498        tokio::select! {
6499            result = &mut run => result,
6500            _ = &mut shutdown => {
6501                stop.store(true, Ordering::SeqCst);
6502                run.await
6503            }
6504        }
6505    }
6506
6507    fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6508        let mut worker = self.clone();
6509        worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6510            policy: self.retry_policy,
6511            stop,
6512        });
6513        worker
6514    }
6515
6516    async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6517        let registration = self.register().await?;
6518        if !registration.registered {
6519            return Err(Error::WorkerLoop(format!(
6520                "worker registration for {:?} was not accepted",
6521                self.worker_id
6522            )));
6523        }
6524        let registered_worker_id = registration.worker_id.clone();
6525        let primary = self.run_registered_until(stop, registration).await;
6526        let deregistration = self
6527            .client
6528            .deregister_worker_registration(&registered_worker_id)
6529            .await;
6530
6531        match (primary, deregistration) {
6532            (Ok(()), Ok(_)) => Ok(()),
6533            (Ok(()), Err(deregistration)) => Err(deregistration),
6534            (Err(primary), Ok(_)) => Err(primary),
6535            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6536                primary: Box::new(primary),
6537                deregistration: Box::new(deregistration),
6538            }),
6539        }
6540    }
6541
6542    async fn run_registered_until(
6543        &self,
6544        stop: Arc<AtomicBool>,
6545        registration: RegisterWorkerResponse,
6546    ) -> Result<()> {
6547        let heartbeat_interval = Duration::from_secs(
6548            registration
6549                .heartbeat_interval_seconds
6550                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6551        );
6552        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
6553        // from the completion of the preceding attempt, including its bounded
6554        // retries. A fixed-epoch interval can leave an already-due tick queued
6555        // while an acknowledgement is slow, producing a catch-up heartbeat as
6556        // soon as that request completes.
6557        let heartbeat = tokio::time::sleep(Duration::ZERO);
6558        tokio::pin!(heartbeat);
6559        // Poll responses may already have leased server-side work by the time
6560        // they become ready, so each poller owns its responses through
6561        // completion or failure instead of racing raw polls in this select.
6562        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6563            let worker = self.clone();
6564            let stop = Arc::clone(&stop);
6565            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6566        });
6567        let mut activity_poller = (!self.activities.is_empty()).then(|| {
6568            let worker = self.clone();
6569            let stop = Arc::clone(&stop);
6570            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6571        });
6572        let mut query_poller = (!self.queries.is_empty()).then(|| {
6573            let worker = self.clone();
6574            let stop = Arc::clone(&stop);
6575            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6576        });
6577
6578        loop {
6579            tokio::select! {
6580                _ = wait_for_worker_stop(&stop) => {
6581                    stop.store(true, Ordering::SeqCst);
6582                    break;
6583                }
6584                _ = &mut heartbeat => {
6585                    let result = self.retry_worker_operation(|| {
6586                        self.client.heartbeat_worker(
6587                            &self.worker_id,
6588                            self.max_concurrent_workflow_tasks,
6589                            self.max_concurrent_activity_tasks,
6590                        )
6591                    }).await;
6592                    heartbeat
6593                        .as_mut()
6594                        .reset(tokio::time::Instant::now() + heartbeat_interval);
6595                    match result {
6596                        Ok(acknowledgement) => {
6597                            if let Some(observer) = &self.heartbeat_observer {
6598                                observer(&WorkerHeartbeatObservation {
6599                                    worker_id: self.worker_id.clone(),
6600                                    task_queue: self.task_queue.clone(),
6601                                    acknowledged_at_unix_millis: SystemTime::now()
6602                                        .duration_since(UNIX_EPOCH)
6603                                        .unwrap_or_default()
6604                                        .as_millis()
6605                                        .min(u64::MAX as u128)
6606                                        as u64,
6607                                    acknowledgement,
6608                                });
6609                            }
6610                        }
6611                        Err(error) => {
6612                            stop.store(true, Ordering::SeqCst);
6613                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6614                            return Err(error);
6615                        }
6616                    }
6617                }
6618                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6619                    workflow_poller = None;
6620                    let stopped_by_server = stop.load(Ordering::SeqCst);
6621                    stop.store(true, Ordering::SeqCst);
6622                    let poller_result = optional_poller_result("workflow", result);
6623                    let join_result =
6624                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6625                    poller_result?;
6626                    join_result?;
6627                    if stopped_by_server {
6628                        return Ok(());
6629                    }
6630                    return Err(Error::WorkerLoop(
6631                        "workflow poller stopped unexpectedly".to_string(),
6632                    ));
6633                }
6634                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6635                    activity_poller = None;
6636                    let stopped_by_server = stop.load(Ordering::SeqCst);
6637                    stop.store(true, Ordering::SeqCst);
6638                    let poller_result = optional_poller_result("activity", result);
6639                    let join_result =
6640                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6641                    poller_result?;
6642                    join_result?;
6643                    if stopped_by_server {
6644                        return Ok(());
6645                    }
6646                    return Err(Error::WorkerLoop(
6647                        "activity poller stopped unexpectedly".to_string(),
6648                    ));
6649                }
6650                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6651                    query_poller = None;
6652                    let stopped_by_server = stop.load(Ordering::SeqCst);
6653                    stop.store(true, Ordering::SeqCst);
6654                    let poller_result = optional_poller_result("query", result);
6655                    let join_result =
6656                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6657                    poller_result?;
6658                    join_result?;
6659                    if stopped_by_server {
6660                        return Ok(());
6661                    }
6662                    return Err(Error::WorkerLoop(
6663                        "query poller stopped unexpectedly".to_string(),
6664                    ));
6665                }
6666            }
6667        }
6668
6669        join_pollers(
6670            workflow_poller.take(),
6671            activity_poller.take(),
6672            query_poller.take(),
6673        )
6674        .await
6675    }
6676
6677    /// Poll and settle at most one task from each enabled task family.
6678    ///
6679    /// A workflow may reach its server-enforced run deadline while this worker
6680    /// holds a task. When the completion endpoint authoritatively rejects that
6681    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
6682    /// terminal `run_status=failed`, the workflow tick is considered settled:
6683    /// the late command was not recorded and cannot replace the terminal run.
6684    /// Every other completion rejection remains an error. This worker-level
6685    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
6686    /// only bounds how long a client waits for a result.
6687    ///
6688    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
6689    /// the original [`Error::Http`] status and response body.
6690    pub async fn run_once(&self) -> Result<usize> {
6691        let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
6692        let mut handled = 0;
6693        match worker.poll_workflow_once().await? {
6694            ManagedPollOutcome::Handled => handled += 1,
6695            ManagedPollOutcome::Stop => return Ok(handled),
6696            ManagedPollOutcome::Idle => {}
6697        }
6698        match worker.poll_activity_once().await? {
6699            ManagedPollOutcome::Handled => handled += 1,
6700            ManagedPollOutcome::Stop => return Ok(handled),
6701            ManagedPollOutcome::Idle => {}
6702        }
6703        if !self.queries.is_empty() {
6704            match worker.poll_query_once().await? {
6705                ManagedPollOutcome::Handled => handled += 1,
6706                ManagedPollOutcome::Stop => return Ok(handled),
6707                ManagedPollOutcome::Idle => {}
6708            }
6709        }
6710        Ok(handled)
6711    }
6712
6713    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6714        let poll_request_id = unique_request_id("rust-workflow-poll");
6715        let response = self
6716            .retry_worker_operation(|| {
6717                self.client.poll_workflow_task_response_with_request_id(
6718                    &self.worker_id,
6719                    &self.task_queue,
6720                    self.poll_timeout,
6721                    &poll_request_id,
6722                    0,
6723                )
6724            })
6725            .await;
6726        let Some(response) = self.settle_worker_poll_response(response).await? else {
6727            return Ok(ManagedPollOutcome::Idle);
6728        };
6729        if response.outcome().should_stop() {
6730            return Ok(ManagedPollOutcome::Stop);
6731        }
6732        let memo_updates_supported =
6733            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6734        let Some(task) = response.task else {
6735            return Ok(ManagedPollOutcome::Idle);
6736        };
6737
6738        let task_id = task.task_id.clone();
6739        let attempt = task.workflow_task_attempt;
6740        let run_id = task.run_id.clone();
6741        let lease_owner = task
6742            .lease_owner
6743            .clone()
6744            .unwrap_or_else(|| self.worker_id.clone());
6745
6746        match self.execute_workflow_task_decision(task) {
6747            Ok(decision)
6748                if commands_use_workflow_memo_updates(&decision.commands)
6749                    && !memo_updates_supported =>
6750            {
6751                self.client
6752                    .fail_workflow_task(
6753                        &task_id,
6754                        &lease_owner,
6755                        attempt,
6756                        Error::WorkflowMemoUpdatesUnavailable.to_string(),
6757                    )
6758                    .await?;
6759            }
6760            Ok(decision) if decision.commands.is_empty() => {
6761                // A replay can consume a recorded pending durable command
6762                // without producing a new command. The standalone protocol
6763                // acknowledges that state through the typed waiting outcome;
6764                // an empty completion is rejected by servers that require at
6765                // least one executable command.
6766                self.client
6767                    .fail_workflow_task_with_type(
6768                        &task_id,
6769                        &lease_owner,
6770                        attempt,
6771                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6772                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6773                    )
6774                    .await?;
6775            }
6776            Ok(decision) => {
6777                let completion = self
6778                    .client
6779                    .complete_workflow_task_with_message_streams(
6780                        &task_id,
6781                        &lease_owner,
6782                        attempt,
6783                        decision.commands,
6784                        decision.message_stream_cursors,
6785                        decision.message_stream_waits,
6786                    )
6787                    .await;
6788                if let Err(error) = completion {
6789                    if !workflow_task_completion_is_terminal_timeout(
6790                        &error,
6791                        &task_id,
6792                        attempt,
6793                        run_id.as_deref(),
6794                    ) {
6795                        return Err(error);
6796                    }
6797                }
6798            }
6799            Err(error) => {
6800                self.client
6801                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6802                    .await?;
6803            }
6804        }
6805
6806        Ok(ManagedPollOutcome::Handled)
6807    }
6808
6809    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6810        while !stop.load(Ordering::SeqCst) {
6811            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6812                stop.store(true, Ordering::SeqCst);
6813                break;
6814            }
6815        }
6816
6817        Ok(())
6818    }
6819
6820    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6821        let poll_request_id = unique_request_id("rust-activity-poll");
6822        let response = self
6823            .retry_worker_operation(|| {
6824                self.client.poll_activity_task_response_with_request_id(
6825                    &self.worker_id,
6826                    &self.task_queue,
6827                    self.poll_timeout,
6828                    &poll_request_id,
6829                    0,
6830                )
6831            })
6832            .await;
6833        let Some(response) = self.settle_worker_poll_response(response).await? else {
6834            return Ok(ManagedPollOutcome::Idle);
6835        };
6836        if response.outcome().should_stop() {
6837            return Ok(ManagedPollOutcome::Stop);
6838        }
6839        let Some(task) = response.task else {
6840            return Ok(ManagedPollOutcome::Idle);
6841        };
6842
6843        let task_id = task.task_id.clone();
6844        let attempt_id = task
6845            .activity_attempt_id
6846            .clone()
6847            .or(task.attempt_id.clone())
6848            .unwrap_or_default();
6849        let lease_owner = task
6850            .lease_owner
6851            .clone()
6852            .unwrap_or_else(|| self.worker_id.clone());
6853        let codec = task.payload_codec.clone();
6854        let result = self.execute_activity_task(task).await;
6855        match result {
6856            Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
6857            Ok(value) => {
6858                let completion = self
6859                    .client
6860                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6861                    .await;
6862                if let Err(error) = completion {
6863                    if !activity_task_rejection_is_final(&error) {
6864                        return Err(error);
6865                    }
6866                }
6867            }
6868            Err(error) => {
6869                let failure = self
6870                    .client
6871                    .fail_activity_task(
6872                        &task_id,
6873                        &attempt_id,
6874                        &lease_owner,
6875                        error.to_string(),
6876                        false,
6877                    )
6878                    .await;
6879                if let Err(error) = failure {
6880                    if !activity_task_rejection_is_final(&error) {
6881                        return Err(error);
6882                    }
6883                }
6884            }
6885        }
6886
6887        Ok(ManagedPollOutcome::Handled)
6888    }
6889
6890    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6891        while !stop.load(Ordering::SeqCst) {
6892            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6893                stop.store(true, Ordering::SeqCst);
6894                break;
6895            }
6896        }
6897
6898        Ok(())
6899    }
6900
6901    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6902        let poll_request_id = unique_request_id("rust-query-poll");
6903        let response = self
6904            .retry_worker_operation(|| {
6905                self.client.poll_query_task_response_with_request_id(
6906                    &self.worker_id,
6907                    &self.task_queue,
6908                    self.poll_timeout,
6909                    &poll_request_id,
6910                    0,
6911                )
6912            })
6913            .await;
6914        let Some(response) = self.settle_worker_poll_response(response).await? else {
6915            return Ok(ManagedPollOutcome::Idle);
6916        };
6917        if response.outcome().should_stop() {
6918            return Ok(ManagedPollOutcome::Stop);
6919        }
6920        let Some(task) = response.task else {
6921            return Ok(ManagedPollOutcome::Idle);
6922        };
6923
6924        let query_task_id = task.query_task_id.clone();
6925        let attempt = task.query_task_attempt;
6926        let lease_owner = task
6927            .lease_owner
6928            .clone()
6929            .unwrap_or_else(|| self.worker_id.clone());
6930        let codec = task.payload_codec.clone();
6931
6932        match self.execute_query_task(task).await {
6933            Ok(value) => {
6934                let result_envelope = match encode_typed_envelope(&value, &codec) {
6935                    Ok(result_envelope) => result_envelope,
6936                    Err(error) => {
6937                        let failure = self
6938                            .client
6939                            .fail_query_task(
6940                                &query_task_id,
6941                                &lease_owner,
6942                                attempt,
6943                                error.to_string(),
6944                                "query_result_encode_failed",
6945                                "QueryResultEncodeFailed",
6946                            )
6947                            .await;
6948                        if let Err(error) = failure {
6949                            if !query_task_rejection_is_final(&error) {
6950                                return Err(error);
6951                            }
6952                        }
6953                        return Ok(ManagedPollOutcome::Handled);
6954                    }
6955                };
6956
6957                if let Err(error) = self
6958                    .client
6959                    .complete_query_task_with_envelope(
6960                        &query_task_id,
6961                        &lease_owner,
6962                        attempt,
6963                        value.clone().into_json()?,
6964                        result_envelope,
6965                    )
6966                    .await
6967                {
6968                    if !query_task_rejection_is_final(&error) {
6969                        return Err(error);
6970                    }
6971                }
6972            }
6973            Err(failure) => {
6974                let result = self
6975                    .client
6976                    .fail_query_task(
6977                        &query_task_id,
6978                        &lease_owner,
6979                        attempt,
6980                        failure.message,
6981                        failure.reason,
6982                        failure.failure_type,
6983                    )
6984                    .await;
6985                if let Err(error) = result {
6986                    if !query_task_rejection_is_final(&error) {
6987                        return Err(error);
6988                    }
6989                }
6990            }
6991        }
6992
6993        Ok(ManagedPollOutcome::Handled)
6994    }
6995
6996    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6997        while !stop.load(Ordering::SeqCst) {
6998            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
6999                stop.store(true, Ordering::SeqCst);
7000                break;
7001            }
7002        }
7003
7004        Ok(())
7005    }
7006
7007    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7008    where
7009        F: FnMut() -> Fut,
7010        Fut: Future<Output = Result<T>>,
7011    {
7012        let mut retries = 0;
7013
7014        loop {
7015            match operation().await {
7016                Err(error)
7017                    if worker_operation_is_retryable(&error)
7018                        && retries < self.retry_policy.max_retries =>
7019                {
7020                    retries += 1;
7021                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7022                }
7023                result => return result,
7024            }
7025        }
7026    }
7027
7028    async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7029        match response {
7030            Ok(response) => Ok(Some(response)),
7031            Err(error) => {
7032                let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7033                    return Err(error);
7034                };
7035                let minimum_delay = self
7036                    .retry_policy
7037                    .initial_backoff
7038                    .max(Duration::from_millis(1));
7039                let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7040                tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7041                Ok(None)
7042            }
7043        }
7044    }
7045
7046    async fn execute_query_task(
7047        &self,
7048        mut task: QueryTask,
7049    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7050        validate_query_task_payloads(&task).map_err(|error| {
7051            QueryTaskExecutionFailure::new(
7052                "query_payload_decode_failed",
7053                error.to_string(),
7054                "QueryPayloadDecodeFailed",
7055            )
7056        })?;
7057
7058        if !self.workflows.contains_key(&task.workflow_type) {
7059            return Err(QueryTaskExecutionFailure::new(
7060                "query_workflow_type_not_registered",
7061                format!("no workflow registered for type {:?}", task.workflow_type),
7062                "WorkflowTypeNotRegistered",
7063            ));
7064        }
7065
7066        let Some(handlers) = self.queries.get(&task.workflow_type) else {
7067            return Err(QueryTaskExecutionFailure::new(
7068                "query_handler_unavailable",
7069                format!(
7070                    "query handlers are unavailable for workflow type {:?}",
7071                    task.workflow_type
7072                ),
7073                "QueryHandlerUnavailable",
7074            ));
7075        };
7076        let Some(query) = handlers.get(&task.query_name) else {
7077            return Err(QueryTaskExecutionFailure::new(
7078                "rejected_unknown_query",
7079                format!("unknown query {:?}", task.query_name),
7080                "QueryFailed",
7081            ));
7082        };
7083
7084        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7085            .map_err(|error| {
7086            QueryTaskExecutionFailure::new(
7087                "query_payload_decode_failed",
7088                format!("cannot decode query arguments: {error}"),
7089                "QueryPayloadDecodeFailed",
7090            )
7091        })?;
7092        let workflow_input_typed =
7093            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7094                .map_err(|error| {
7095                    QueryTaskExecutionFailure::new(
7096                        "query_workflow_state_unavailable",
7097                        format!("cannot decode workflow start input: {error}"),
7098                        "QueryWorkflowStateUnavailable",
7099                    )
7100                })?;
7101        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7102            QueryTaskExecutionFailure::new(
7103                "query_workflow_state_unavailable",
7104                format!("cannot project workflow start input: {error}"),
7105                "QueryWorkflowStateUnavailable",
7106            )
7107        })?;
7108        hydrate_query_history_from_export(&mut task).map_err(|error| {
7109            QueryTaskExecutionFailure::new(
7110                "query_workflow_state_unavailable",
7111                format!("cannot restore query history snapshot: {error}"),
7112                "QueryWorkflowStateUnavailable",
7113            )
7114        })?;
7115        enrich_query_history_from_export(&mut task).map_err(|error| {
7116            QueryTaskExecutionFailure::new(
7117                "query_workflow_state_unavailable",
7118                format!("cannot restore compact query history payloads: {error}"),
7119                "QueryWorkflowStateUnavailable",
7120            )
7121        })?;
7122        let signal_events = query_signal_events(&task).map_err(|error| {
7123            QueryTaskExecutionFailure::new(
7124                "query_workflow_state_unavailable",
7125                format!("cannot decode committed workflow signals: {error}"),
7126                "QueryWorkflowStateUnavailable",
7127            )
7128        })?;
7129        let history_events = Arc::new(std::mem::take(&mut task.history_events));
7130        let context = QueryContext {
7131            workflow_id: task.workflow_id,
7132            run_id: task.run_id,
7133            workflow_type: task.workflow_type.clone(),
7134            run_status: task.run_status,
7135            workflow_input,
7136            workflow_input_avro_value: workflow_input_typed.clone(),
7137            history_events: Arc::clone(&history_events),
7138            signal_events: Arc::new(signal_events),
7139        };
7140
7141        let future = match query {
7142            RegisteredQuery::Snapshot(handler) => handler(context, args),
7143            RegisteredQuery::Replayed {
7144                state_type,
7145                handler,
7146            } => {
7147                let workflow = self
7148                    .workflows
7149                    .get(&task.workflow_type)
7150                    .expect("workflow registration was checked above");
7151                if workflow.state_type != Some(*state_type) {
7152                    return Err(QueryTaskExecutionFailure::new(
7153                        "query_workflow_state_unavailable",
7154                        "replayed query state type does not match its workflow registration",
7155                        "QueryWorkflowStateUnavailable",
7156                    ));
7157                }
7158                let replay = workflow.replay.as_ref().ok_or_else(|| {
7159                    QueryTaskExecutionFailure::new(
7160                        "query_workflow_state_unavailable",
7161                        format!(
7162                            "workflow type {:?} is not registered for instance-state replay",
7163                            task.workflow_type
7164                        ),
7165                        "QueryWorkflowStateUnavailable",
7166                    )
7167                })?;
7168                let workflow_state = Arc::new(Mutex::new(
7169                    WorkflowState::new_with_identity(
7170                        history_events.as_ref().clone(),
7171                        context.workflow_id.clone(),
7172                        context.run_id.clone(),
7173                        self.task_queue.clone(),
7174                        task.payload_codec,
7175                        None,
7176                    )
7177                    .map_err(|error| {
7178                        QueryTaskExecutionFailure::new(
7179                            "query_workflow_state_unavailable",
7180                            format!("workflow replay failed before query: {error}"),
7181                            "QueryWorkflowStateUnavailable",
7182                        )
7183                    })?,
7184                ));
7185                let workflow_context = WorkflowContext {
7186                    state: workflow_state,
7187                };
7188                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7189                let mut cx = TaskContext::from_waker(noop_waker_ref());
7190                match invocation.future.as_mut().poll(&mut cx) {
7191                    Poll::Ready(Ok(_)) => {
7192                        workflow_context
7193                            .ensure_history_consumed()
7194                            .map_err(|error| {
7195                                QueryTaskExecutionFailure::new(
7196                                    "query_workflow_state_unavailable",
7197                                    format!("workflow replay failed before query: {error}"),
7198                                    "QueryWorkflowStateUnavailable",
7199                                )
7200                            })?;
7201                    }
7202                    Poll::Ready(Err(error)) => {
7203                        return Err(QueryTaskExecutionFailure::new(
7204                            "query_workflow_state_unavailable",
7205                            format!("workflow replay failed before query: {error}"),
7206                            "QueryWorkflowStateUnavailable",
7207                        ));
7208                    }
7209                    Poll::Pending => {
7210                        let commands = workflow_context.take_commands().map_err(|error| {
7211                            QueryTaskExecutionFailure::new(
7212                                "query_workflow_state_unavailable",
7213                                format!("workflow replay failed before query: {error}"),
7214                                "QueryWorkflowStateUnavailable",
7215                            )
7216                        })?;
7217                        if commands.is_empty()
7218                            && !workflow_context
7219                                .matched_recorded_pending()
7220                                .map_err(|error| {
7221                                    QueryTaskExecutionFailure::new(
7222                                        "query_workflow_state_unavailable",
7223                                        format!("workflow replay failed before query: {error}"),
7224                                        "QueryWorkflowStateUnavailable",
7225                                    )
7226                                })?
7227                        {
7228                            return Err(QueryTaskExecutionFailure::new(
7229                                "query_workflow_state_unavailable",
7230                                "workflow replay yielded without a durable command",
7231                                "QueryWorkflowStateUnavailable",
7232                            ));
7233                        }
7234                    }
7235                }
7236                let state = (invocation.snapshot)().map_err(|error| {
7237                    QueryTaskExecutionFailure::new(
7238                        "query_workflow_state_unavailable",
7239                        format!("cannot snapshot replayed workflow state: {error}"),
7240                        "QueryWorkflowStateUnavailable",
7241                    )
7242                })?;
7243                handler(context, state, args).map_err(|message| {
7244                    QueryTaskExecutionFailure::new(
7245                        "query_workflow_state_unavailable",
7246                        message,
7247                        "QueryWorkflowStateUnavailable",
7248                    )
7249                })?
7250            }
7251        };
7252
7253        future.await.map_err(|error| {
7254            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7255        })
7256    }
7257
7258    #[cfg(test)]
7259    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7260        Ok(self.execute_workflow_task_decision(task)?.commands)
7261    }
7262
7263    fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7264        validate_workflow_task_payloads(&task)?;
7265
7266        if let Some(update_id) = task
7267            .workflow_update_id
7268            .as_deref()
7269            .filter(|update_id| !update_id.is_empty())
7270        {
7271            return self
7272                .execute_update_task(&task, update_id)
7273                .map(WorkflowTaskDecision::without_message_streams);
7274        }
7275
7276        let workflow = self
7277            .workflows
7278            .get(&task.workflow_type)
7279            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7280        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7281        let resume_signal = decode_resume_signal(&task)?;
7282        let history_budget = WorkflowHistoryBudget {
7283            event_count: task
7284                .total_history_events
7285                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7286            size_bytes: task.history_size_bytes,
7287            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7288            pressure: task.history_budget_pressure.clone(),
7289        };
7290        let workflow_command_identity = task
7291            .workflow_command_id
7292            .clone()
7293            .filter(|identity| !identity.is_empty())
7294            .unwrap_or_default();
7295        let mut workflow_state = WorkflowState::new_with_identity(
7296            task.history_events,
7297            task.workflow_id,
7298            task.run_id,
7299            self.task_queue.clone(),
7300            task.payload_codec.clone(),
7301            resume_signal,
7302        )?;
7303        workflow_state.history_budget = history_budget;
7304        workflow_state.workflow_command_identity = workflow_command_identity;
7305        workflow_state.cancel_requested = task.cancel_requested;
7306        let state = Arc::new(Mutex::new(workflow_state));
7307        let ctx = WorkflowContext { state };
7308        let mut future = (workflow.execute)(ctx.clone(), input);
7309        let mut cx = TaskContext::from_waker(noop_waker_ref());
7310
7311        match future.as_mut().poll(&mut cx) {
7312            Poll::Ready(Ok(result)) => {
7313                ctx.ensure_history_consumed()?;
7314                let result = encode_typed_envelope(&result, &task.payload_codec)?;
7315                let mut commands = ctx.take_commands()?;
7316                commands.push(json!({
7317                    "type": "complete_workflow",
7318                    "result": result
7319                }));
7320                self.message_stream_decision(&ctx, commands)
7321            }
7322            Poll::Ready(Err(error)) => {
7323                if let Error::ContinueAsNew(request) = error {
7324                    let mut commands = ctx.take_commands()?;
7325                    if let Some(command) = ctx.continue_as_new_command(request)? {
7326                        commands.push(command);
7327                    }
7328                    ctx.ensure_history_consumed()?;
7329                    return self.message_stream_decision(&ctx, commands);
7330                }
7331                if workflow_task_integrity_error(&error) {
7332                    // Replay and protocol failures describe the workflow-task
7333                    // decision itself. Preserve their specific failure reason
7334                    // instead of replacing it with the derivative fact that
7335                    // recorded commands remain unconsumed.
7336                    return Err(error);
7337                }
7338                // A handler error must not hide a committed durable command that
7339                // upgraded workflow code no longer consumes.
7340                ctx.ensure_history_consumed()?;
7341                let mut commands = ctx.take_commands()?;
7342                commands.push(workflow_failure_command(&error));
7343                self.message_stream_decision(&ctx, commands)
7344            }
7345            Poll::Pending => {
7346                let commands = ctx.take_commands()?;
7347                if commands.is_empty() && !ctx.matched_recorded_pending()? {
7348                    Err(Error::WorkflowYieldedWithoutCommand)
7349                } else {
7350                    self.message_stream_decision(&ctx, commands)
7351                }
7352            }
7353        }
7354    }
7355
7356    fn message_stream_decision(
7357        &self,
7358        ctx: &WorkflowContext,
7359        commands: Vec<Value>,
7360    ) -> Result<WorkflowTaskDecision> {
7361        let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7362        Ok(WorkflowTaskDecision {
7363            commands,
7364            message_stream_cursors,
7365            message_stream_waits,
7366        })
7367    }
7368
7369    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7370        if !self.workflows.contains_key(&task.workflow_type) {
7371            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7372        }
7373
7374        let accepted = task.history_events.iter().rev().find_map(|event| {
7375            (event.event_type == "UpdateAccepted"
7376                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7377            .then_some(&event.payload)
7378        });
7379        let update_name = accepted
7380            .and_then(|payload| payload.get("update_name"))
7381            .and_then(Value::as_str)
7382            .or(task.update_name.as_deref())
7383            .unwrap_or_default();
7384        let Some(handler) = self
7385            .updates
7386            .get(&task.workflow_type)
7387            .and_then(|handlers| handlers.get(update_name))
7388        else {
7389            return Ok(vec![json!({
7390                "type": "fail_update",
7391                "update_id": update_id,
7392                "message": format!(
7393                    "no update handler is registered for {}.{update_name}",
7394                    task.workflow_type
7395                ),
7396                "exception_type": "UnknownUpdate",
7397                "non_retryable": true,
7398            })]);
7399        };
7400        let arguments = accepted
7401            .and_then(|payload| payload.get("arguments"))
7402            .or(task.arguments.as_ref());
7403        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7404        let context = QueryContext {
7405            workflow_id: task.workflow_id.clone(),
7406            run_id: task.run_id.clone(),
7407            workflow_type: task.workflow_type.clone(),
7408            run_status: Some("running".to_string()),
7409            workflow_input: Value::Null,
7410            workflow_input_avro_value: AvroValue::Null,
7411            history_events: Arc::new(task.history_events.clone()),
7412            signal_events: Arc::new(Vec::new()),
7413        };
7414        let mut future = handler(context, arguments);
7415        let mut cx = TaskContext::from_waker(noop_waker_ref());
7416
7417        match future.as_mut().poll(&mut cx) {
7418            Poll::Ready(Ok(result)) => Ok(vec![json!({
7419                "type": "complete_update",
7420                "update_id": update_id,
7421                "result": encode_typed_envelope(&result, &task.payload_codec)?,
7422            })]),
7423            Poll::Ready(Err(error)) => Ok(vec![json!({
7424                "type": "fail_update",
7425                "update_id": update_id,
7426                "message": error.to_string(),
7427                "exception_type": "UpdateFailed",
7428                "non_retryable": true,
7429            })]),
7430            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7431        }
7432    }
7433
7434    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7435        validate_activity_task_payloads(&task)?;
7436
7437        let handler = self
7438            .activities
7439            .get(&task.activity_type)
7440            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7441        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7442        let attempt_id = task
7443            .activity_attempt_id
7444            .clone()
7445            .or(task.attempt_id.clone())
7446            .unwrap_or_default();
7447        let lease_owner = task
7448            .lease_owner
7449            .clone()
7450            .unwrap_or_else(|| self.worker_id.clone());
7451        let ctx = ActivityContext {
7452            client: self.client.clone(),
7453            task_id: task.task_id,
7454            activity_attempt_id: attempt_id,
7455            lease_owner,
7456            activity_type: task.activity_type,
7457            attempt_number: task.attempt_number,
7458            task_queue: self.task_queue.clone(),
7459            worker_id: self.worker_id.clone(),
7460        };
7461
7462        handler(ctx, args).await
7463    }
7464}
7465
7466fn poller_result(
7467    kind: &str,
7468    result: std::result::Result<Result<()>, tokio::task::JoinError>,
7469) -> Result<()> {
7470    match result {
7471        Ok(result) => result,
7472        Err(error) => Err(Error::WorkerLoop(format!(
7473            "{kind} poller join error: {error}"
7474        ))),
7475    }
7476}
7477
7478fn optional_poller_result(
7479    kind: &str,
7480    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7481) -> Result<()> {
7482    match result {
7483        Some(result) => poller_result(kind, result),
7484        None => Ok(()),
7485    }
7486}
7487
7488async fn join_pollers(
7489    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7490    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7491    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7492) -> Result<()> {
7493    let mut first_error = None;
7494
7495    if let Some(handle) = workflow_poller {
7496        if let Err(error) = poller_result("workflow", handle.await) {
7497            first_error.get_or_insert(error);
7498        }
7499    }
7500
7501    if let Some(handle) = activity_poller {
7502        if let Err(error) = poller_result("activity", handle.await) {
7503            first_error.get_or_insert(error);
7504        }
7505    }
7506
7507    if let Some(handle) = query_poller {
7508        if let Err(error) = poller_result("query", handle.await) {
7509            first_error.get_or_insert(error);
7510        }
7511    }
7512
7513    if let Some(error) = first_error {
7514        Err(error)
7515    } else {
7516        Ok(())
7517    }
7518}
7519
7520fn default_worker_id() -> String {
7521    let millis = SystemTime::now()
7522        .duration_since(UNIX_EPOCH)
7523        .unwrap_or_default()
7524        .as_millis();
7525    format!("rust-worker-{}-{millis}", std::process::id())
7526}
7527
7528fn percent_encode_path_segment(segment: &str) -> String {
7529    const HEX: &[u8; 16] = b"0123456789ABCDEF";
7530    let mut encoded = String::with_capacity(segment.len());
7531
7532    for byte in segment.bytes() {
7533        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7534            encoded.push(char::from(byte));
7535        } else {
7536            encoded.push('%');
7537            encoded.push(char::from(HEX[(byte >> 4) as usize]));
7538            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7539        }
7540    }
7541
7542    encoded
7543}
7544
7545fn unique_request_id(prefix: &str) -> String {
7546    let nanos = SystemTime::now()
7547        .duration_since(UNIX_EPOCH)
7548        .unwrap_or_default()
7549        .as_nanos();
7550    format!("{prefix}-{}-{nanos}", std::process::id())
7551}
7552
7553#[derive(Debug)]
7554struct QueryTaskExecutionFailure {
7555    reason: String,
7556    message: String,
7557    failure_type: String,
7558}
7559
7560impl QueryTaskExecutionFailure {
7561    fn new(
7562        reason: impl Into<String>,
7563        message: impl Into<String>,
7564        failure_type: impl Into<String>,
7565    ) -> Self {
7566        Self {
7567            reason: reason.into(),
7568            message: message.into(),
7569            failure_type: failure_type.into(),
7570        }
7571    }
7572}
7573
7574/// Typed local state owned by one deterministic workflow invocation.
7575///
7576/// Use [`WorkflowInstance::update`] for the same state transitions during
7577/// ordinary execution and replay. A replayed query receives a detached
7578/// immutable `Arc<S>` rather than this mutation-capable handle.
7579#[derive(Clone, Debug)]
7580pub struct WorkflowInstance<S> {
7581    state: Arc<Mutex<S>>,
7582}
7583
7584impl<S> WorkflowInstance<S> {
7585    fn new(state: S) -> Self {
7586        Self {
7587            state: Arc::new(Mutex::new(state)),
7588        }
7589    }
7590
7591    /// Read the current workflow-instance state without changing it.
7592    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7593        let state = self
7594            .state
7595            .lock()
7596            .map_err(|_| Error::WorkflowStatePoisoned)?;
7597        Ok(reader(&state))
7598    }
7599
7600    /// Apply one deterministic workflow-instance state transition.
7601    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7602        let mut state = self
7603            .state
7604            .lock()
7605            .map_err(|_| Error::WorkflowStatePoisoned)?;
7606        Ok(transition(&mut state))
7607    }
7608}
7609
7610impl<S: Clone> WorkflowInstance<S> {
7611    fn snapshot(&self) -> Result<S> {
7612        self.read(Clone::clone)
7613    }
7614}
7615
7616#[derive(Clone, Debug, PartialEq)]
7617pub struct MessageStreamMessage {
7618    pub stream_name: String,
7619    pub message_id: String,
7620    pub position: u64,
7621    pub arguments: Vec<AvroValue>,
7622}
7623
7624#[derive(Clone, Debug)]
7625pub struct MessageStream {
7626    ctx: WorkflowContext,
7627    name: String,
7628}
7629
7630impl MessageStream {
7631    /// Wait for one message, then return a bounded currently-available batch.
7632    pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7633        if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7634            return Err(Error::Codec(format!(
7635                "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7636            )));
7637        }
7638        loop {
7639            if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7640                return Ok(batch);
7641            }
7642
7643            self.ctx.record_message_stream_wait(&self.name)?;
7644            let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7645            let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7646            self.ctx.buffer_message_stream_delivery(arguments)?;
7647            if let Some(sequence) = replay_wait_sequence {
7648                self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7649            }
7650        }
7651    }
7652
7653    pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7654        self.receive(1)
7655            .await?
7656            .into_iter()
7657            .next()
7658            .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7659    }
7660}
7661
7662#[derive(Clone, Debug)]
7663pub struct WorkflowContext {
7664    state: Arc<Mutex<WorkflowState>>,
7665}
7666
7667fn valid_memo_key(key: &str) -> bool {
7668    let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7669
7670    !key.is_empty()
7671        && key.len() <= 64
7672        && (numeric_candidate.is_empty()
7673            || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7674        && key
7675            .bytes()
7676            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7677}
7678
7679fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7680    BASE64
7681        .decode(encode_avro_value(value)?.blob)
7682        .map(|bytes| bytes.len())
7683        .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7684}
7685
7686fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7687    let AvroValue::Map(entries) = value else {
7688        return Err(Error::InvalidMemoUpdate(
7689            "entries must serialize to an Avro string-keyed map".to_string(),
7690        ));
7691    };
7692    if require_entries && entries.is_empty() {
7693        return Err(Error::InvalidMemoUpdate(
7694            "at least one entry is required".to_string(),
7695        ));
7696    }
7697    if entries.len() > MAX_MEMO_ENTRIES {
7698        return Err(Error::InvalidMemoUpdate(format!(
7699            "at most {MAX_MEMO_ENTRIES} entries are allowed"
7700        )));
7701    }
7702
7703    for (key, value) in &entries {
7704        if !valid_memo_key(&key) {
7705            return Err(Error::InvalidMemoUpdate(
7706                "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7707            ));
7708        }
7709        if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7710            return Err(Error::InvalidMemoUpdate(format!(
7711                "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7712            )));
7713        }
7714    }
7715
7716    let value = AvroValue::Map(entries);
7717    if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7718        return Err(Error::InvalidMemoUpdate(format!(
7719            "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7720        )));
7721    }
7722    Ok(value)
7723}
7724
7725fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7726    let object = envelope.as_object().ok_or_else(|| {
7727        Error::InvalidMemoUpdate(
7728            "history field must use the public {codec, blob} payload envelope".to_string(),
7729        )
7730    })?;
7731    if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7732        return Err(Error::InvalidMemoUpdate(
7733            "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7734        ));
7735    }
7736
7737    canonical_memo_entries(
7738        decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7739        require_entries,
7740    )
7741}
7742
7743impl WorkflowContext {
7744    pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7745        let name = name.into();
7746        if name.is_empty()
7747            || name.len() > 128
7748            || !name.bytes().all(|byte| {
7749                byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7750            })
7751        {
7752            return Err(Error::Codec(
7753                "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7754                    .to_string(),
7755            ));
7756        }
7757        Ok(MessageStream {
7758            ctx: self.clone(),
7759            name,
7760        })
7761    }
7762
7763    fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7764        let mut state = self
7765            .state
7766            .lock()
7767            .map_err(|_| Error::WorkflowStatePoisoned)?;
7768        let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7769        state
7770            .message_stream_waits
7771            .insert(name.to_string(), position);
7772        Ok(())
7773    }
7774
7775    fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7776        let mut state = self
7777            .state
7778            .lock()
7779            .map_err(|_| Error::WorkflowStatePoisoned)?;
7780        let cursor = state
7781            .message_stream_cursors
7782            .get(&message.stream_name)
7783            .copied()
7784            .unwrap_or(0);
7785        if message.position <= cursor {
7786            return Ok(());
7787        }
7788        let pending = state
7789            .message_stream_messages
7790            .entry(message.stream_name.clone())
7791            .or_default();
7792        if pending.iter().any(|candidate| {
7793            candidate.position == message.position || candidate.message_id == message.message_id
7794        }) {
7795            return Ok(());
7796        }
7797        pending.push(message);
7798        pending.sort_by_key(|candidate| candidate.position);
7799        Ok(())
7800    }
7801
7802    fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7803        if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7804            match delivery {
7805                MessageStreamDelivery::Message(message) => {
7806                    let stream_name = message.stream_name.clone();
7807                    self.buffer_message_stream(message)?;
7808                    return Ok(Some(stream_name));
7809                }
7810                MessageStreamDelivery::Cursor {
7811                    stream_name,
7812                    through_position,
7813                } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7814            }
7815        }
7816        Ok(None)
7817    }
7818
7819    fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7820        let state = self
7821            .state
7822            .lock()
7823            .map_err(|_| Error::WorkflowStatePoisoned)?;
7824        Ok(match state.recorded_commands.get(state.command_cursor) {
7825            Some(RecordedCommand::SignalWait {
7826                sequence,
7827                signal_name,
7828                ..
7829            }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7830            _ => None,
7831        })
7832    }
7833
7834    fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7835        let (history, payload_codec) = {
7836            let state = self
7837                .state
7838                .lock()
7839                .map_err(|_| Error::WorkflowStatePoisoned)?;
7840            (
7841                Arc::clone(&state.history_events),
7842                state.payload_codec.clone(),
7843            )
7844        };
7845
7846        let Some(opened_index) = history.iter().position(|event| {
7847            event.event_type == "SignalWaitOpened"
7848                && durable_event_sequence(event) == Some(wait_sequence)
7849                && event.payload.get("signal_name").and_then(Value::as_str)
7850                    == Some(MESSAGE_STREAM_SIGNAL)
7851        }) else {
7852            return Ok(());
7853        };
7854        let boundary_index = history
7855            .iter()
7856            .enumerate()
7857            .skip(opened_index + 1)
7858            .find_map(|(index, event)| {
7859                (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7860                    && is_authored_command_open_event(event))
7861                .then_some(index)
7862            })
7863            .unwrap_or(history.len());
7864
7865        for event in history[opened_index + 1..boundary_index]
7866            .iter()
7867            .filter(|event| {
7868                event.event_type == "SignalReceived"
7869                    && event.payload.get("signal_name").and_then(Value::as_str)
7870                        == Some(MESSAGE_STREAM_SIGNAL)
7871            })
7872        {
7873            let arguments = decode_signal_event_arguments(event, &payload_codec)?
7874                .into_iter()
7875                .map(AvroValue::into_json)
7876                .collect::<Result<Vec<_>>>()?;
7877            self.buffer_message_stream_delivery(arguments)?;
7878        }
7879        Ok(())
7880    }
7881
7882    fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7883        let mut state = self
7884            .state
7885            .lock()
7886            .map_err(|_| Error::WorkflowStatePoisoned)?;
7887        let cursor = state
7888            .message_stream_cursors
7889            .entry(name.to_string())
7890            .or_default();
7891        *cursor = (*cursor).max(through_position);
7892        if let Some(pending) = state.message_stream_messages.get_mut(name) {
7893            pending.retain(|message| message.position > through_position);
7894        }
7895        Ok(())
7896    }
7897
7898    fn take_message_stream_batch(
7899        &self,
7900        name: &str,
7901        max_items: usize,
7902    ) -> Result<Option<Vec<MessageStreamMessage>>> {
7903        let mut state = self
7904            .state
7905            .lock()
7906            .map_err(|_| Error::WorkflowStatePoisoned)?;
7907        let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7908        let pending = state
7909            .message_stream_messages
7910            .entry(name.to_string())
7911            .or_default();
7912        let count = contiguous_message_stream_count(pending, cursor, max_items);
7913        if count == 0 {
7914            return Ok(None);
7915        }
7916        let batch = pending.drain(..count).collect::<Vec<_>>();
7917        let position = batch.last().map(|message| message.position).unwrap_or(0);
7918        state
7919            .message_stream_cursors
7920            .insert(name.to_string(), position);
7921        state.message_stream_waits.remove(name);
7922        Ok(Some(batch))
7923    }
7924
7925    fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7926        let state = self
7927            .state
7928            .lock()
7929            .map_err(|_| Error::WorkflowStatePoisoned)?;
7930        let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7931        cursors.sort_by_key(|(name, _)| *name);
7932        let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7933        waits.sort_by_key(|(name, _)| *name);
7934        Ok((
7935            cursors
7936                .into_iter()
7937                .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7938                .collect(),
7939            waits
7940                .into_iter()
7941                .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7942                .collect(),
7943        ))
7944    }
7945    /// Identity of the parent workflow currently being replayed.
7946    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7947        let state = self
7948            .state
7949            .lock()
7950            .map_err(|_| Error::WorkflowStatePoisoned)?;
7951        Ok(WorkflowIdentity {
7952            workflow_id: state.workflow_id.clone(),
7953            run_id: state.run_id.clone(),
7954        })
7955    }
7956
7957    /// Return the server-published history budget for this workflow task.
7958    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7959        let state = self
7960            .state
7961            .lock()
7962            .map_err(|_| Error::WorkflowStatePoisoned)?;
7963        Ok(state.history_budget.clone())
7964    }
7965
7966    /// Continue this workflow instance as a fresh run with replacement arguments.
7967    ///
7968    /// Return this value directly from the workflow handler. The worker converts
7969    /// it to the terminal protocol command only after replay has consumed every
7970    /// recorded durable command.
7971    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7972        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7973    }
7974
7975    /// Continue as new with optional workflow-type and task-queue overrides.
7976    pub fn continue_as_new_with_options<T: Serialize>(
7977        &self,
7978        options: ContinueAsNewOptions,
7979        args: T,
7980    ) -> Result<Value> {
7981        options.validate()?;
7982        Err(Error::ContinueAsNew(ContinueAsNewRequest {
7983            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
7984            options,
7985        }))
7986    }
7987
7988    pub fn activity<T: Serialize>(
7989        &self,
7990        activity_type: impl Into<String>,
7991        args: T,
7992    ) -> ActivityCall {
7993        self.activity_with_options(activity_type, ActivityOptions::new(), args)
7994    }
7995
7996    pub fn activity_on_queue<T, Q>(
7997        &self,
7998        activity_type: impl Into<String>,
7999        task_queue: Option<Q>,
8000        args: T,
8001    ) -> ActivityCall
8002    where
8003        T: Serialize,
8004        Q: Into<String>,
8005    {
8006        let mut options = ActivityOptions::new();
8007        options.task_queue = task_queue.map(Into::into);
8008        self.activity_with_options(activity_type, options, args)
8009    }
8010
8011    /// Schedule one durable activity with retry, routing, and timeout options.
8012    ///
8013    /// Options are validated before the command is emitted. Once the command is
8014    /// recorded, replay consumes the same activity lifecycle at this command
8015    /// position and never emits a duplicate schedule.
8016    ///
8017    /// ```no_run
8018    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
8019    /// # use std::time::Duration;
8020    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
8021    /// let result = ctx
8022    ///     .activity_with_options(
8023    ///         "charge-card",
8024    ///         ActivityOptions::new()
8025    ///             .task_queue("payments")
8026    ///             .retry_policy(
8027    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
8028    ///                     Duration::from_secs(1),
8029    ///                     2,
8030    ///                     Some(Duration::from_secs(30)),
8031    ///                 ),
8032    ///             )
8033    ///             .start_to_close_timeout(Duration::from_secs(60))
8034    ///             .schedule_to_close_timeout(Duration::from_secs(180))
8035    ///             .heartbeat_timeout(Duration::from_secs(15)),
8036    ///         json!([{"order_id": "order-42"}]),
8037    ///     )
8038    ///     .await;
8039    /// match result {
8040    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
8041    ///         "reason": failure.reason,
8042    ///         "timeout_kind": failure.timeout_kind,
8043    ///     })),
8044    ///     other => other,
8045    /// }
8046    /// # }
8047    /// ```
8048    pub fn activity_with_options<T: Serialize>(
8049        &self,
8050        activity_type: impl Into<String>,
8051        options: ActivityOptions,
8052        args: T,
8053    ) -> ActivityCall {
8054        ActivityCall {
8055            ctx: self.clone(),
8056            activity_type: activity_type.into(),
8057            options,
8058            args: Some(AvroValue::from_serialize(&args)),
8059            scheduled: false,
8060            parallel_group_path: Vec::new(),
8061        }
8062    }
8063
8064    pub async fn activity_avro_value<T: Serialize>(
8065        &self,
8066        activity_type: impl Into<String>,
8067        args: T,
8068    ) -> Result<AvroValue> {
8069        let mut call = self.activity(activity_type, args);
8070        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8071    }
8072
8073    pub async fn activity_avro_value_with_options<T: Serialize>(
8074        &self,
8075        activity_type: impl Into<String>,
8076        options: ActivityOptions,
8077        args: T,
8078    ) -> Result<AvroValue> {
8079        let mut call = self.activity_with_options(activity_type, options, args);
8080        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8081    }
8082
8083    /// Schedule an activity with a Serde request and decode its Serde result.
8084    pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8085    where
8086        I: Serialize,
8087        O: DeserializeOwned,
8088    {
8089        self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8090            .await
8091    }
8092
8093    /// Schedule an activity with options and decode its result into `O`.
8094    ///
8095    /// Both directions use the fixed Avro Value codec. In particular, this
8096    /// method does not deserialize the JSON-safe inspection projection returned
8097    /// by the dynamic [`ActivityCall`] future.
8098    pub async fn activity_typed_with_options<I, O>(
8099        &self,
8100        activity_type: impl Into<String>,
8101        options: ActivityOptions,
8102        args: I,
8103    ) -> Result<O>
8104    where
8105        I: Serialize,
8106        O: DeserializeOwned,
8107    {
8108        let activity_type = activity_type.into();
8109        let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8110            handler_type_error::<I>(
8111                HandlerKind::Activity,
8112                &activity_type,
8113                HandlerValueKind::Input,
8114                error.to_string(),
8115            )
8116        });
8117        let mut call = ActivityCall {
8118            ctx: self.clone(),
8119            activity_type: activity_type.clone(),
8120            options,
8121            args: Some(encoded),
8122            scheduled: false,
8123            parallel_group_path: Vec::new(),
8124        };
8125        let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8126        decode_handler_result(result, HandlerKind::Activity, &activity_type)
8127    }
8128
8129    /// Schedule and join a deterministic activity/child/timer group.
8130    ///
8131    /// Nested groups retain their input shape. Every durable leaf is scheduled
8132    /// before this future yields, results are assembled by declaration order,
8133    /// and a failure returns [`Error::ParallelFailed`] with typed cause,
8134    /// declaration path, stable group metadata, and completed siblings.
8135    pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8136        ParallelCall::new(self.clone(), operations)
8137    }
8138
8139    /// Alias for [`WorkflowContext::parallel`].
8140    pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8141        self.parallel(operations)
8142    }
8143
8144    /// Lossless fixed-Avro variant of [`WorkflowContext::parallel`].
8145    pub async fn parallel_avro_value(
8146        &self,
8147        operations: Vec<ParallelOperation>,
8148    ) -> Result<Vec<ParallelAvroResult>> {
8149        let mut call = self.parallel(operations);
8150        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8151    }
8152
8153    /// Start every durable operation and resume from the one winner persisted
8154    /// by Server. Non-winning operations continue and remain addressable.
8155    pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8156        let operations = operations
8157            .into_iter()
8158            .enumerate()
8159            .map(|(index, operation)| (SelectionKey::Index(index), operation))
8160            .collect();
8161        SelectCall::new(self.clone(), operations)
8162    }
8163
8164    /// Named-key variant of [`WorkflowContext::select`].
8165    pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8166    where
8167        K: Into<SelectionKey>,
8168    {
8169        SelectCall::new(
8170            self.clone(),
8171            operations
8172                .into_iter()
8173                .map(|(key, operation)| (key.into(), operation))
8174                .collect(),
8175        )
8176    }
8177
8178    /// Create a workflow-local deterministic compensation registry.
8179    pub fn saga(&self) -> Saga {
8180        Saga::new(self.clone())
8181    }
8182
8183    /// Whether the current workflow task carries a cooperative cancel request.
8184    pub fn is_cancellation_requested(&self) -> Result<bool> {
8185        let state = self
8186            .state
8187            .lock()
8188            .map_err(|_| Error::WorkflowStatePoisoned)?;
8189        Ok(state.cancel_requested)
8190    }
8191
8192    /// Raise a typed cooperative cancellation at an author-controlled point.
8193    ///
8194    /// Passing this result to [`Saga::finish`] compensates already registered
8195    /// forward steps before the cancellation remains the initiating outcome.
8196    pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8197        if self.is_cancellation_requested()? {
8198            return Err(Error::WorkflowCancellationRequested(
8199                WorkflowCancellationRequested,
8200            ));
8201        }
8202        Ok(())
8203    }
8204
8205    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8206        SignalCall {
8207            ctx: self.clone(),
8208            signal_name: signal_name.into(),
8209            runtime_reserved_allowed: false,
8210            opened_wait: false,
8211            matched_pending: false,
8212            parallel_group_path: Vec::new(),
8213        }
8214    }
8215
8216    fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8217        SignalCall {
8218            ctx: self.clone(),
8219            signal_name: signal_name.into(),
8220            runtime_reserved_allowed: true,
8221            opened_wait: false,
8222            matched_pending: false,
8223            parallel_group_path: Vec::new(),
8224        }
8225    }
8226
8227    pub async fn wait_signal_avro_value(
8228        &self,
8229        signal_name: impl Into<String>,
8230    ) -> Result<Vec<AvroValue>> {
8231        let mut call = self.wait_signal(signal_name);
8232        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8233    }
8234
8235    /// Return every committed signal argument list with the given name.
8236    ///
8237    /// This history-backed view is deterministic and is intended for
8238    /// condition predicates that must be re-evaluated after a signal while the
8239    /// workflow is blocked on [`WorkflowContext::wait_condition`].
8240    pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8241        self.signals_avro_value(signal_name)?
8242            .into_iter()
8243            .map(|arguments| {
8244                arguments
8245                    .into_iter()
8246                    .map(AvroValue::into_json)
8247                    .collect::<Result<Vec<_>>>()
8248            })
8249            .collect()
8250    }
8251
8252    /// Lossless fixed Avro Value view of committed signals with the given name.
8253    pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8254        let state = self
8255            .state
8256            .lock()
8257            .map_err(|_| Error::WorkflowStatePoisoned)?;
8258        state
8259            .history_events
8260            .iter()
8261            .filter(|event| {
8262                event.event_type == "SignalReceived"
8263                    && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8264            })
8265            .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8266            .collect()
8267    }
8268
8269    /// Return every committed update argument list with the given name.
8270    ///
8271    /// Accepted and applied records for the same update ID are de-duplicated.
8272    /// A Server task created after an update therefore replays the workflow and
8273    /// re-evaluates an open condition without application polling.
8274    pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8275        self.updates_avro_value(update_name)?
8276            .into_iter()
8277            .map(|arguments| {
8278                arguments
8279                    .into_iter()
8280                    .map(AvroValue::into_json)
8281                    .collect::<Result<Vec<_>>>()
8282            })
8283            .collect()
8284    }
8285
8286    /// Lossless fixed Avro Value view of committed updates with the given name.
8287    pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8288        let state = self
8289            .state
8290            .lock()
8291            .map_err(|_| Error::WorkflowStatePoisoned)?;
8292        let mut seen = Vec::new();
8293        let mut updates = Vec::new();
8294        for event in state.history_events.iter() {
8295            if !matches!(
8296                event.event_type.as_str(),
8297                "UpdateAccepted" | "UpdateApplied"
8298            ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8299                || event.payload.get("arguments").is_none()
8300            {
8301                continue;
8302            }
8303            if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8304                if seen.iter().any(|recorded| recorded == update_id) {
8305                    continue;
8306                }
8307                seen.push(update_id.to_string());
8308            }
8309            updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8310        }
8311        Ok(updates)
8312    }
8313
8314    /// Wait for a deterministic predicate to become true or for its durable
8315    /// timeout to elapse.
8316    ///
8317    /// Prefer [`wait_condition!`] for inline predicates so changes to the Rust
8318    /// predicate tokens automatically change the recorded definition
8319    /// fingerprint. Direct callers must provide an equally stable identity in
8320    /// [`ConditionWaitOptions`].
8321    pub fn wait_condition<F>(
8322        &self,
8323        options: ConditionWaitOptions,
8324        predicate: F,
8325    ) -> ConditionWaitCall
8326    where
8327        F: Fn() -> Result<bool> + Send + 'static,
8328    {
8329        ConditionWaitCall {
8330            ctx: self.clone(),
8331            options,
8332            predicate: Box::new(predicate),
8333            occurrence_id: None,
8334            opened_wait: false,
8335            parallel_group_path: Vec::new(),
8336        }
8337    }
8338
8339    /// Wait for server-backed durable time without blocking the worker executor.
8340    ///
8341    /// Polling this future emits one `start_timer` command and yields. The
8342    /// server records the deadline, so neither worker nor server restarts reset
8343    /// the wait. Replay resolves the future only from a `TimerScheduled` and
8344    /// `TimerFired` pair at the same position in the shared durable-command
8345    /// stream, with matching sequence, timer identity, and delay. Sub-second
8346    /// durations round up because protocol deadlines use whole seconds.
8347    ///
8348    /// ```no_run
8349    /// # use durable_workflow::{json, Client, Worker};
8350    /// # use std::time::Duration;
8351    /// # fn configure(client: Client) {
8352    /// let mut worker = Worker::new(client, "rust-workers");
8353    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
8354    ///     ctx.sleep(Duration::from_secs(5)).await?;
8355    ///     Ok(json!({"status": "timer fired"}))
8356    /// });
8357    /// # }
8358    /// ```
8359    pub fn sleep(&self, duration: Duration) -> TimerCall {
8360        let delay_seconds = duration
8361            .as_secs()
8362            .checked_add(u64::from(duration.subsec_nanos() > 0));
8363        TimerCall {
8364            ctx: self.clone(),
8365            delay_seconds,
8366            scheduled: false,
8367            matched_pending: false,
8368            parallel_group_path: Vec::new(),
8369        }
8370    }
8371
8372    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
8373    pub fn start_timer(&self, duration: Duration) -> TimerCall {
8374        self.sleep(duration)
8375    }
8376
8377    /// Evaluate a non-deterministic callback once and durably record its typed value.
8378    ///
8379    /// On replay the callback is not invoked: the value is decoded from the
8380    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
8381    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
8382    /// small values that must remain fixed for the lifetime of a workflow run.
8383    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8384    where
8385        T: Serialize + DeserializeOwned,
8386        F: FnOnce() -> T,
8387    {
8388        {
8389            let mut state = self
8390                .state
8391                .lock()
8392                .map_err(|_| Error::WorkflowStatePoisoned)?;
8393            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8394                return match recorded {
8395                    RecordedCommand::SideEffect { sequence, value } => {
8396                        state.command_cursor += 1;
8397                        value.deserialize().map_err(|error| {
8398                            Error::NonDeterministicReplay(ReplayFailure::new(
8399                                "side_effect_type_mismatch",
8400                                Some(sequence),
8401                                Some(std::any::type_name::<T>().to_string()),
8402                                Some(error.to_string()),
8403                                "recorded side-effect value is incompatible with the requested Rust type",
8404                            ))
8405                        })
8406                    }
8407                    other => Err(command_mismatch(&other, "side effect")),
8408                };
8409            }
8410        }
8411
8412        let value = callback();
8413        let avro_value = AvroValue::from_serialize(&value)?;
8414        let mut state = self
8415            .state
8416            .lock()
8417            .map_err(|_| Error::WorkflowStatePoisoned)?;
8418        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8419        state.commands.push(json!({
8420            "type": "record_side_effect",
8421            "result": result,
8422        }));
8423        Ok(value)
8424    }
8425
8426    /// Record or replay a lossless fixed Avro Value side effect.
8427    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8428    where
8429        F: FnOnce() -> AvroValue,
8430    {
8431        {
8432            let mut state = self
8433                .state
8434                .lock()
8435                .map_err(|_| Error::WorkflowStatePoisoned)?;
8436            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8437                return match recorded {
8438                    RecordedCommand::SideEffect { value, .. } => {
8439                        state.command_cursor += 1;
8440                        Ok(value)
8441                    }
8442                    other => Err(command_mismatch(&other, "side effect")),
8443                };
8444            }
8445        }
8446
8447        let value = callback();
8448        let mut state = self
8449            .state
8450            .lock()
8451            .map_err(|_| Error::WorkflowStatePoisoned)?;
8452        let result = encode_typed_envelope(&value, &state.payload_codec)?;
8453        state.commands.push(json!({
8454            "type": "record_side_effect",
8455            "result": result,
8456        }));
8457        Ok(value)
8458    }
8459
8460    /// Append output items at a deterministic workflow command boundary.
8461    ///
8462    /// Stable idempotency keys are derived from the server-provided durable
8463    /// workflow command identity, command ordinal, and item index. Replay
8464    /// consumes the recorded side effect and never emits another append.
8465    pub fn append_workflow_stream(
8466        &self,
8467        stream_name: impl Into<String>,
8468        items: &[WorkflowStreamAppendItem],
8469        max_pending_items: Option<u64>,
8470    ) -> Result<()> {
8471        if items.is_empty() {
8472            return Err(Error::Codec(
8473                "workflow_stream_items_empty: append requires at least one item".to_string(),
8474            ));
8475        }
8476        if max_pending_items == Some(0) {
8477            return Err(Error::Codec(
8478                "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8479                    .to_string(),
8480            ));
8481        }
8482        let stream_name = stream_name.into();
8483        if stream_name.is_empty() {
8484            return Err(Error::Codec(
8485                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8486            ));
8487        }
8488
8489        let mut state = self
8490            .state
8491            .lock()
8492            .map_err(|_| Error::WorkflowStatePoisoned)?;
8493        let command_ordinal = state.workflow_stream_command_counter;
8494        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8495            state.workflow_stream_command_counter += 1;
8496            return match recorded {
8497                RecordedCommand::SideEffect { .. } => {
8498                    state.command_cursor += 1;
8499                    Ok(())
8500                }
8501                other => Err(command_mismatch(&other, "workflow stream append")),
8502            };
8503        }
8504
8505        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8506        state.workflow_stream_command_counter += 1;
8507        let wire_items = items
8508            .iter()
8509            .enumerate()
8510            .map(|(item_index, item)| {
8511                item.wire_value(Some(format!(
8512                    "dw-stream:{identity}:{command_ordinal}:{item_index}"
8513                )))
8514            })
8515            .collect::<Vec<_>>();
8516        let mut directive = json!({
8517            "operation": "append",
8518            "stream_name": stream_name,
8519            "command_identity": identity,
8520            "command_ordinal": command_ordinal,
8521            "items": wire_items,
8522        });
8523        if let Some(max_pending_items) = max_pending_items {
8524            directive["max_pending_items"] = json!(max_pending_items);
8525        }
8526        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8527        state.commands.push(json!({
8528            "type": "record_side_effect",
8529            "result": result,
8530            "workflow_stream": directive,
8531        }));
8532        Ok(())
8533    }
8534
8535    /// Close a run-scoped output stream at a deterministic command boundary.
8536    pub fn close_workflow_stream(
8537        &self,
8538        stream_name: impl Into<String>,
8539        retention_seconds: Option<u64>,
8540    ) -> Result<()> {
8541        self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8542    }
8543
8544    /// Mark a run-scoped output stream errored at a deterministic command boundary.
8545    pub fn error_workflow_stream(
8546        &self,
8547        stream_name: impl Into<String>,
8548        error_reason: impl Into<String>,
8549        retention_seconds: Option<u64>,
8550    ) -> Result<()> {
8551        let error_reason = error_reason.into();
8552        if error_reason.is_empty() {
8553            return Err(Error::Codec(
8554                "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8555            ));
8556        }
8557        self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8558    }
8559
8560    fn finish_workflow_stream(
8561        &self,
8562        stream_name: String,
8563        error_reason: Option<String>,
8564        retention_seconds: Option<u64>,
8565    ) -> Result<()> {
8566        if stream_name.is_empty() {
8567            return Err(Error::Codec(
8568                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8569            ));
8570        }
8571        if retention_seconds == Some(0) {
8572            return Err(Error::Codec(
8573                "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8574            ));
8575        }
8576        let mut state = self
8577            .state
8578            .lock()
8579            .map_err(|_| Error::WorkflowStatePoisoned)?;
8580        let command_ordinal = state.workflow_stream_command_counter;
8581        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8582            state.workflow_stream_command_counter += 1;
8583            return match recorded {
8584                RecordedCommand::SideEffect { .. } => {
8585                    state.command_cursor += 1;
8586                    Ok(())
8587                }
8588                other => Err(command_mismatch(&other, "workflow stream close")),
8589            };
8590        }
8591        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8592        state.workflow_stream_command_counter += 1;
8593        let mut directive = json!({
8594            "operation": if error_reason.is_some() { "error" } else { "close" },
8595            "stream_name": stream_name,
8596            "command_identity": identity,
8597            "command_ordinal": command_ordinal,
8598        });
8599        if let Some(error_reason) = error_reason {
8600            directive["error_reason"] = json!(error_reason);
8601        }
8602        if let Some(retention_seconds) = retention_seconds {
8603            directive["retention_seconds"] = json!(retention_seconds);
8604        }
8605        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8606        state.commands.push(json!({
8607            "type": "record_side_effect",
8608            "result": result,
8609            "workflow_stream": directive,
8610        }));
8611        Ok(())
8612    }
8613
8614    fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8615        let identity = state.workflow_command_identity.as_str();
8616        if identity.is_empty() {
8617            return Err(Error::MissingWorkflowCommandIdentity);
8618        }
8619        Ok(identity)
8620    }
8621
8622    /// Validate, emit, or replay a typed workflow search-attribute update.
8623    ///
8624    /// The command is non-blocking within a workflow decision, but its
8625    /// `SearchAttributesUpserted` event occupies the same deterministic command
8626    /// stream as activities, timers, conditions, and other durable operations.
8627    pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8628        update.validate()?;
8629        let (attributes, attribute_types) = update.into_wire_parts();
8630        let mut state = self
8631            .state
8632            .lock()
8633            .map_err(|_| Error::WorkflowStatePoisoned)?;
8634
8635        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8636            return match recorded {
8637                RecordedCommand::SearchAttributes {
8638                    sequence,
8639                    attributes: recorded_attributes,
8640                    attribute_types: recorded_attribute_types,
8641                } => {
8642                    if recorded_attributes != attributes {
8643                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8644                            "search_attribute_value_mismatch",
8645                            Some(sequence),
8646                            Some(recorded_attributes.to_string()),
8647                            Some(attributes.to_string()),
8648                            "search-attribute values differ from the recorded durable command",
8649                        )));
8650                    }
8651                    if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8652                        if recorded_types != attribute_types {
8653                            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8654                                "search_attribute_type_mismatch",
8655                                Some(sequence),
8656                                Some(json!(recorded_types).to_string()),
8657                                Some(json!(attribute_types).to_string()),
8658                                "search-attribute declared types differ from the recorded durable command",
8659                            )));
8660                        }
8661                    }
8662                    state.command_cursor += 1;
8663                    Ok(())
8664                }
8665                other => Err(command_mismatch(&other, "search-attribute update")),
8666            };
8667        }
8668
8669        let mut command = serde_json::Map::from_iter([
8670            ("type".to_string(), json!("upsert_search_attributes")),
8671            ("attributes".to_string(), attributes),
8672        ]);
8673        if !attribute_types.is_empty() {
8674            command.insert("attribute_types".to_string(), json!(attribute_types));
8675        }
8676        state.commands.push(Value::Object(command));
8677        Ok(())
8678    }
8679
8680    /// Record a UUIDv4 once and return the same UUID on every replay.
8681    pub fn uuid_v4(&self) -> Result<Uuid> {
8682        self.side_effect(Uuid::new_v4)
8683    }
8684
8685    /// Select the newest supported version for a change, or replay the version
8686    /// already committed for that stable change ID.
8687    pub fn get_version(
8688        &self,
8689        change_id: impl Into<String>,
8690        min_supported: i32,
8691        max_supported: i32,
8692    ) -> Result<i32> {
8693        let change_id = change_id.into();
8694        if change_id.trim().is_empty() {
8695            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8696                "version_change_id_invalid",
8697                None,
8698                Some("non-empty change ID".to_string()),
8699                Some(change_id),
8700                "version markers require a stable non-empty change ID",
8701            )));
8702        }
8703        if min_supported > max_supported {
8704            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8705                "version_range_invalid",
8706                None,
8707                Some("min_supported <= max_supported".to_string()),
8708                Some(format!("{min_supported}..={max_supported}")),
8709                "version marker supported range is invalid",
8710            )));
8711        }
8712
8713        let mut state = self
8714            .state
8715            .lock()
8716            .map_err(|_| Error::WorkflowStatePoisoned)?;
8717        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8718            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8719            return Ok(version);
8720        }
8721
8722        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8723            return match recorded {
8724                RecordedCommand::VersionMarker {
8725                    sequence,
8726                    change_id: recorded_change_id,
8727                    version,
8728                    ..
8729                } => {
8730                    if recorded_change_id != change_id {
8731                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8732                            "version_change_id_mismatch",
8733                            Some(sequence),
8734                            Some(recorded_change_id),
8735                            Some(change_id),
8736                            "recorded version marker change ID differs from current workflow code",
8737                        )));
8738                    }
8739                    ensure_version_supported(
8740                        &change_id,
8741                        version,
8742                        min_supported,
8743                        max_supported,
8744                        sequence,
8745                    )?;
8746                    state.command_cursor += 1;
8747                    state.version_markers.insert(change_id, (version, sequence));
8748                    Ok(version)
8749                }
8750                other => Err(command_mismatch(
8751                    &other,
8752                    format!("version marker:{change_id}"),
8753                )),
8754            };
8755        }
8756
8757        let version = max_supported;
8758        state.commands.push(json!({
8759            "type": "record_version_marker",
8760            "change_id": change_id,
8761            "version": version,
8762            "min_supported": min_supported,
8763            "max_supported": max_supported,
8764        }));
8765        // Sequence numbers are assigned by the server. Zero identifies a marker
8766        // selected in this uncommitted decision batch for duplicate-call checks.
8767        state.version_markers.insert(change_id, (version, 0));
8768        Ok(version)
8769    }
8770
8771    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
8772    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8773        Ok(self.get_version(change_id, -1, 1)? == 1)
8774    }
8775
8776    /// Keep a patch marker in history after the legacy branch has been removed.
8777    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8778        self.get_version(change_id, -1, 1).map(|_| ())
8779    }
8780
8781    /// Merge non-indexed workflow memo metadata through durable history.
8782    ///
8783    /// Avro `null` deletes a key. The SDK encodes the complete patch in the
8784    /// public Avro payload envelope consumed by Server and Cloud runtimes.
8785    pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8786        let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8787        let mut state = self
8788            .state
8789            .lock()
8790            .map_err(|_| Error::WorkflowStatePoisoned)?;
8791
8792        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8793            return match recorded {
8794                RecordedCommand::Memo {
8795                    sequence,
8796                    entries: recorded_entries,
8797                } => {
8798                    if recorded_entries != entries {
8799                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8800                            "memo_update_mismatch",
8801                            Some(sequence),
8802                            Some(format!("{recorded_entries:?}")),
8803                            Some(format!("{entries:?}")),
8804                            "recorded memo entries differ from the current workflow update",
8805                        )));
8806                    }
8807                    state.command_cursor += 1;
8808                    Ok(())
8809                }
8810                other => Err(command_mismatch(&other, "memo upsert")),
8811            };
8812        }
8813
8814        let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8815        state.commands.push(json!({
8816            "type": "upsert_memo",
8817            "entries": entries_envelope,
8818        }));
8819        Ok(())
8820    }
8821
8822    /// Start a named durable child on an explicit queue and await its result.
8823    ///
8824    /// The command is recorded in the parent's sequence-ordered durable command
8825    /// stream. Replay keeps a scheduled child pending without emitting another
8826    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
8827    /// values preserve the history payload codec and include both sides of the
8828    /// durable relationship; failures are returned as
8829    /// [`Error::ChildWorkflowFailed`].
8830    ///
8831    /// ```no_run
8832    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
8833    /// # fn configure(client: Client) {
8834    /// let mut worker = Worker::new(client, "parent-workers");
8835    /// worker.register_workflow("order-parent", |ctx, _input| async move {
8836    ///     let child = ctx
8837    ///         .start_child_workflow(
8838    ///             "fulfil-order",
8839    ///             ChildWorkflowOptions::new("fulfilment-workers")
8840    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
8841    ///             json!([{"order_id": "order-42"}]),
8842    ///         )
8843    ///         .await?;
8844    ///     Ok(child.result)
8845    /// });
8846    /// # }
8847    /// ```
8848    pub fn start_child_workflow<T: Serialize>(
8849        &self,
8850        workflow_type: impl Into<String>,
8851        options: ChildWorkflowOptions,
8852        args: T,
8853    ) -> ChildWorkflowCall {
8854        ChildWorkflowCall {
8855            ctx: self.clone(),
8856            workflow_type: workflow_type.into(),
8857            options,
8858            args: Some(AvroValue::from_serialize(&args)),
8859            scheduled: false,
8860            matched_pending: false,
8861            parallel_group_path: Vec::new(),
8862        }
8863    }
8864
8865    pub async fn start_child_workflow_avro_value<T: Serialize>(
8866        &self,
8867        workflow_type: impl Into<String>,
8868        options: ChildWorkflowOptions,
8869        args: T,
8870    ) -> Result<ChildWorkflowAvroResult> {
8871        let mut call = self.start_child_workflow(workflow_type, options, args);
8872        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8873    }
8874
8875    fn take_commands(&self) -> Result<Vec<Value>> {
8876        let mut state = self
8877            .state
8878            .lock()
8879            .map_err(|_| Error::WorkflowStatePoisoned)?;
8880        Ok(std::mem::take(&mut state.commands))
8881    }
8882
8883    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8884        let mut state = self
8885            .state
8886            .lock()
8887            .map_err(|_| Error::WorkflowStatePoisoned)?;
8888
8889        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8890            return Err(command_mismatch(&recorded, "continue as new"));
8891        }
8892        if state.recorded_continue_as_new_sequence.is_some() {
8893            state.continue_as_new_consumed = true;
8894            return Ok(None);
8895        }
8896
8897        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8898        let mut command = serde_json::Map::from_iter([
8899            ("type".to_string(), json!("continue_as_new")),
8900            ("arguments".to_string(), arguments),
8901            ("queue".to_string(), json!(state.task_queue.clone())),
8902        ]);
8903        if let Some(workflow_type) = request.options.workflow_type {
8904            command.insert("workflow_type".to_string(), json!(workflow_type));
8905        }
8906        if let Some(task_queue) = request.options.task_queue {
8907            command.insert("queue".to_string(), json!(task_queue));
8908        }
8909        Ok(Some(Value::Object(command)))
8910    }
8911
8912    fn matched_recorded_pending(&self) -> Result<bool> {
8913        let state = self
8914            .state
8915            .lock()
8916            .map_err(|_| Error::WorkflowStatePoisoned)?;
8917        Ok(state.matched_recorded_pending)
8918    }
8919
8920    fn ensure_history_consumed(&self) -> Result<()> {
8921        let state = self
8922            .state
8923            .lock()
8924            .map_err(|_| Error::WorkflowStatePoisoned)?;
8925        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8926            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8927                "recorded_commands_unconsumed",
8928                Some(command.sequence()),
8929                Some(command.shape().to_string()),
8930                Some("workflow completion".to_string()),
8931                "workflow completed before consuming all recorded durable commands",
8932            )));
8933        }
8934        if let Some(sequence) = state
8935            .recorded_continue_as_new_sequence
8936            .filter(|_| !state.continue_as_new_consumed)
8937        {
8938            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8939                "recorded_continue_as_new_unconsumed",
8940                Some(sequence),
8941                Some("continue as new".to_string()),
8942                Some("workflow completion".to_string()),
8943                "workflow completed without consuming its recorded continue-as-new transition",
8944            )));
8945        }
8946        Ok(())
8947    }
8948}
8949
8950fn contiguous_message_stream_count(
8951    pending: &[MessageStreamMessage],
8952    cursor: u64,
8953    max_items: usize,
8954) -> usize {
8955    pending
8956        .iter()
8957        .take(max_items)
8958        .enumerate()
8959        .take_while(|(offset, message)| {
8960            u64::try_from(*offset)
8961                .ok()
8962                .and_then(|offset| cursor.checked_add(offset + 1))
8963                == Some(message.position)
8964        })
8965        .count()
8966}
8967
8968fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
8969    matches!(
8970        event.event_type.as_str(),
8971        "ActivityScheduled"
8972            | "TimerScheduled"
8973            | "ChildWorkflowScheduled"
8974            | "SignalWaitOpened"
8975            | "ConditionWaitOpened"
8976            | "SearchAttributesUpserted"
8977            | "SideEffectRecorded"
8978            | "VersionMarkerRecorded"
8979            | "MemoUpserted"
8980            | "WorkflowContinuedAsNew"
8981    )
8982}
8983
8984#[derive(Debug)]
8985struct WorkflowState {
8986    workflow_id: Option<String>,
8987    run_id: Option<String>,
8988    task_queue: String,
8989    payload_codec: String,
8990    history_events: Arc<Vec<HistoryEvent>>,
8991    history_budget: WorkflowHistoryBudget,
8992    cancel_requested: bool,
8993    resume_signal: Option<ResumeSignal>,
8994    recorded_commands: Vec<RecordedCommand>,
8995    selection_markers: Vec<SelectionMarker>,
8996    selection_marker_cursor: usize,
8997    cancelled_selection_members: Vec<SelectionCancellation>,
8998    recorded_continue_as_new_sequence: Option<u64>,
8999    continue_as_new_consumed: bool,
9000    command_cursor: usize,
9001    condition_wait_occurrence_counter: u64,
9002    matched_recorded_pending: bool,
9003    version_markers: HashMap<String, (i32, u64)>,
9004    workflow_command_identity: String,
9005    workflow_stream_command_counter: u64,
9006    commands: Vec<Value>,
9007    message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9008    message_stream_cursors: HashMap<String, u64>,
9009    message_stream_waits: HashMap<String, u64>,
9010}
9011
9012impl WorkflowState {
9013    #[cfg(test)]
9014    fn new(
9015        history: Vec<HistoryEvent>,
9016        task_queue: String,
9017        payload_codec: String,
9018        resume_signal: Option<ResumeSignal>,
9019    ) -> Result<Self> {
9020        Self::new_with_identity(
9021            history,
9022            None,
9023            None,
9024            task_queue,
9025            payload_codec,
9026            resume_signal,
9027        )
9028    }
9029
9030    fn new_with_identity(
9031        history: Vec<HistoryEvent>,
9032        workflow_id: Option<String>,
9033        run_id: Option<String>,
9034        task_queue: String,
9035        payload_codec: String,
9036        resume_signal: Option<ResumeSignal>,
9037    ) -> Result<Self> {
9038        let recorded_commands = recorded_commands(
9039            &history,
9040            &payload_codec,
9041            WorkflowIdentity {
9042                workflow_id: workflow_id.clone(),
9043                run_id: run_id.clone(),
9044            },
9045        )?;
9046        let selection_markers = recorded_selection_markers(&history)?;
9047        let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9048        let recorded_continue_as_new = history
9049            .iter()
9050            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9051            .collect::<Vec<_>>();
9052        if recorded_continue_as_new.len() > 1 {
9053            return Err(invalid_recorded_history(
9054                "duplicate_continue_as_new_transition",
9055                recorded_continue_as_new
9056                    .last()
9057                    .and_then(|event| durable_event_sequence(event))
9058                    .unwrap_or(0),
9059                "one WorkflowContinuedAsNew event",
9060                &format!(
9061                    "{} WorkflowContinuedAsNew events",
9062                    recorded_continue_as_new.len()
9063                ),
9064                "workflow history records one continue-as-new transition more than once",
9065            ));
9066        }
9067        let recorded_continue_as_new_sequence = recorded_continue_as_new
9068            .first()
9069            .map(|event| {
9070                durable_event_sequence(event).ok_or_else(|| {
9071                    Error::NonDeterministicReplay(ReplayFailure::new(
9072                        "continue_as_new_sequence_missing",
9073                        None,
9074                        Some("recorded transition sequence".to_string()),
9075                        Some("missing sequence".to_string()),
9076                        "WorkflowContinuedAsNew history is missing its recorded sequence",
9077                    ))
9078                })
9079            })
9080            .transpose()?;
9081        let mut message_stream_cursors = HashMap::new();
9082        for event in &history {
9083            if !matches!(
9084                event.event_type.as_str(),
9085                "SignalReceived" | "SignalApplied"
9086            ) || event.payload.get("signal_name").and_then(Value::as_str)
9087                != Some(MESSAGE_STREAM_SIGNAL)
9088            {
9089                continue;
9090            }
9091            let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9092            if arguments.len() != 1 {
9093                continue;
9094            }
9095            let envelope = arguments[0].clone().into_json()?;
9096            let Some(envelope) = envelope.as_object() else {
9097                continue;
9098            };
9099            if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9100            {
9101                continue;
9102            }
9103            let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9104                continue;
9105            };
9106            let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9107            else {
9108                continue;
9109            };
9110            let cursor = message_stream_cursors
9111                .entry(stream_name.to_string())
9112                .or_insert(0);
9113            *cursor = (*cursor).max(through_position);
9114        }
9115        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9116        let cancel_requested = history.iter().any(|event| {
9117            matches!(
9118                event.event_type.as_str(),
9119                "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9120            )
9121        });
9122        Ok(Self {
9123            workflow_command_identity: String::new(),
9124            workflow_stream_command_counter: 0,
9125            workflow_id,
9126            run_id,
9127            task_queue,
9128            payload_codec,
9129            history_events: Arc::new(history),
9130            history_budget: WorkflowHistoryBudget {
9131                event_count,
9132                ..WorkflowHistoryBudget::default()
9133            },
9134            cancel_requested,
9135            resume_signal,
9136            recorded_commands,
9137            selection_markers,
9138            selection_marker_cursor: 0,
9139            cancelled_selection_members,
9140            recorded_continue_as_new_sequence,
9141            continue_as_new_consumed: false,
9142            command_cursor: 0,
9143            condition_wait_occurrence_counter: 0,
9144            matched_recorded_pending: false,
9145            version_markers: HashMap::new(),
9146            commands: Vec::new(),
9147            message_stream_messages: HashMap::new(),
9148            message_stream_cursors,
9149            message_stream_waits: HashMap::new(),
9150        })
9151    }
9152}
9153
9154enum MessageStreamDelivery {
9155    Message(MessageStreamMessage),
9156    Cursor {
9157        stream_name: String,
9158        through_position: u64,
9159    },
9160}
9161
9162fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9163    if arguments.len() != 1 {
9164        return Ok(None);
9165    }
9166    let envelope = arguments
9167        .into_iter()
9168        .next()
9169        .expect("one argument was checked");
9170    let Some(envelope) = envelope.as_object() else {
9171        return Ok(None);
9172    };
9173    let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9174        return Ok(None);
9175    };
9176    if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9177        let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9178        else {
9179            return Ok(None);
9180        };
9181        return Ok(Some(MessageStreamDelivery::Cursor {
9182            stream_name: stream_name.to_string(),
9183            through_position,
9184        }));
9185    }
9186    if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9187        return Ok(None);
9188    }
9189    let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9190        return Ok(None);
9191    };
9192    let Some(position) = envelope
9193        .get("position")
9194        .and_then(Value::as_u64)
9195        .filter(|value| *value > 0)
9196    else {
9197        return Ok(None);
9198    };
9199    let Some(payload_envelope) = envelope.get("payload_envelope") else {
9200        return Ok(None);
9201    };
9202    let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9203    else {
9204        return Ok(None);
9205    };
9206    let decoded = decode_avro_value(&payload_envelope)?;
9207    let AvroValue::Array(values) = decoded else {
9208        return Ok(None);
9209    };
9210    Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9211        stream_name: stream_name.to_string(),
9212        message_id: message_id.to_string(),
9213        position,
9214        arguments: values,
9215    })))
9216}
9217
9218#[derive(Clone, Debug)]
9219enum RecordedCommand {
9220    Activity {
9221        sequence: u64,
9222        activity_type: Option<String>,
9223        options: Option<RecordedActivityOptions>,
9224        outcome: Option<ActivityOutcome>,
9225        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9226    },
9227    Timer {
9228        sequence: u64,
9229        delay_seconds: u64,
9230        fired: bool,
9231        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9232    },
9233    ChildWorkflow {
9234        sequence: u64,
9235        workflow_type: Option<String>,
9236        outcome: Option<ChildWorkflowOutcome>,
9237        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9238    },
9239    SignalWait {
9240        sequence: u64,
9241        signal_name: String,
9242        value: Option<Vec<AvroValue>>,
9243        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9244    },
9245    ConditionWait {
9246        sequence: u64,
9247        occurrence_id: String,
9248        condition_key: Option<String>,
9249        predicate_identity: String,
9250        timeout_seconds: Option<u64>,
9251        result: Option<ConditionWaitResult>,
9252        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9253    },
9254    SearchAttributes {
9255        sequence: u64,
9256        attributes: Value,
9257        attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9258    },
9259    SideEffect {
9260        sequence: u64,
9261        value: AvroValue,
9262    },
9263    VersionMarker {
9264        sequence: u64,
9265        change_id: String,
9266        version: i32,
9267    },
9268    Memo {
9269        sequence: u64,
9270        entries: AvroValue,
9271    },
9272}
9273
9274#[derive(Clone, Debug, PartialEq, Eq)]
9275struct SelectionMarker {
9276    selection_group_id: String,
9277    selection_group_base_sequence: u64,
9278    selection_group_size: usize,
9279    member_key: SelectionKey,
9280    member_index: usize,
9281    member_base_sequence: u64,
9282    member_size: usize,
9283    operation_kind: String,
9284    operation_identity: String,
9285    outcome: String,
9286    resolution_event_id: String,
9287    resolution_event_type: String,
9288}
9289
9290#[derive(Clone, Debug, PartialEq, Eq)]
9291struct SelectionCancellation {
9292    selection_group_id: String,
9293    member_key: SelectionKey,
9294    member_index: usize,
9295    member_base_sequence: u64,
9296    member_size: usize,
9297    operation_kind: String,
9298    operation_identity: String,
9299}
9300
9301fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9302    let mut markers: Vec<SelectionMarker> = Vec::new();
9303    for event in events
9304        .iter()
9305        .filter(|event| event.event_type == "SelectionResolved")
9306    {
9307        let payload = &event.payload;
9308        let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9309        let group_size = required_selection_usize(payload, "selection_group_size")?;
9310        let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9311        let member_size = required_selection_usize(payload, "member_size")?;
9312        let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9313        let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9314            invalid_recorded_history(
9315                "selection_marker_invalid",
9316                base_sequence,
9317                "non-empty selection_group_id",
9318                &payload.to_string(),
9319                "selection winner history is missing its durable group identity",
9320            )
9321        })?;
9322        let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9323        if group_id != expected_group_id {
9324            return Err(invalid_recorded_history(
9325                "selection_marker_invalid",
9326                base_sequence,
9327                &expected_group_id,
9328                &group_id,
9329                "selection winner history contains an incompatible group identity",
9330            ));
9331        }
9332        let group_end = base_sequence
9333            .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9334            .unwrap_or(u64::MAX);
9335        let member_end = member_base_sequence
9336            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9337            .unwrap_or(u64::MAX);
9338        if member_index >= group_size
9339            || member_base_sequence < base_sequence
9340            || member_end > group_end
9341        {
9342            return Err(invalid_recorded_history(
9343                "selection_marker_invalid",
9344                base_sequence,
9345                "winner member within selection group bounds",
9346                &payload.to_string(),
9347                "selection winner history contains an invalid member range",
9348            ));
9349        }
9350        let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9351            invalid_recorded_history(
9352                "selection_marker_invalid",
9353                base_sequence,
9354                "selection operation kind",
9355                &payload.to_string(),
9356                "selection winner history is missing its operation kind",
9357            )
9358        })?;
9359        if !matches!(
9360            operation_kind.as_str(),
9361            "activity" | "child" | "timer" | "signal" | "condition" | "group"
9362        ) {
9363            return Err(invalid_recorded_history(
9364                "selection_marker_invalid",
9365                base_sequence,
9366                "activity, child, timer, signal, condition, or group",
9367                &operation_kind,
9368                "selection winner history contains an unsupported operation kind",
9369            ));
9370        }
9371        let operation_identity =
9372            payload_string(payload, "operation_identity").ok_or_else(|| {
9373                invalid_recorded_history(
9374                    "selection_marker_invalid",
9375                    base_sequence,
9376                    "non-empty operation identity",
9377                    &payload.to_string(),
9378                    "selection winner history is missing its durable operation identity",
9379                )
9380            })?;
9381        let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9382            invalid_recorded_history(
9383                "selection_marker_invalid",
9384                base_sequence,
9385                "completed or failed selection outcome",
9386                &payload.to_string(),
9387                "selection winner history is missing its outcome",
9388            )
9389        })?;
9390        if !matches!(outcome.as_str(), "completed" | "failed") {
9391            return Err(invalid_recorded_history(
9392                "selection_marker_invalid",
9393                base_sequence,
9394                "completed or failed selection outcome",
9395                &outcome,
9396                "selection winner history contains an unsupported outcome",
9397            ));
9398        }
9399        let marker = SelectionMarker {
9400            selection_group_id: group_id,
9401            selection_group_base_sequence: base_sequence,
9402            selection_group_size: group_size,
9403            member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9404            member_index,
9405            member_base_sequence,
9406            member_size,
9407            operation_kind,
9408            operation_identity,
9409            outcome,
9410            resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9411                || {
9412                    invalid_recorded_history(
9413                        "selection_marker_invalid",
9414                        base_sequence,
9415                        "durable resolution_event_id",
9416                        &payload.to_string(),
9417                        "selection winner history is missing its terminal event identity",
9418                    )
9419                },
9420            )?,
9421            resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9422                || {
9423                    invalid_recorded_history(
9424                        "selection_marker_invalid",
9425                        base_sequence,
9426                        "durable resolution_event_type",
9427                        &payload.to_string(),
9428                        "selection winner history is missing its terminal event type",
9429                    )
9430                },
9431            )?,
9432        };
9433        if let Some(existing) = markers
9434            .iter()
9435            .find(|existing| existing.selection_group_id == marker.selection_group_id)
9436        {
9437            if existing != &marker {
9438                return Err(invalid_recorded_history(
9439                    "selection_marker_conflict",
9440                    base_sequence,
9441                    &format!("one winner for {}", marker.selection_group_id),
9442                    &payload.to_string(),
9443                    "selection history records conflicting winners for one durable group",
9444                ));
9445            }
9446            continue;
9447        }
9448        markers.push(marker);
9449    }
9450    Ok(markers)
9451}
9452
9453fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9454    let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9455    for event in events
9456        .iter()
9457        .filter(|event| event.event_type == "SelectionOperationCancelled")
9458    {
9459        let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9460            invalid_recorded_history(
9461                "selection_cancellation_invalid",
9462                0,
9463                "non-empty selection_group_id",
9464                &event.payload.to_string(),
9465                "selection cancellation history is missing its group identity",
9466            )
9467        })?;
9468        let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9469        let marker = SelectionCancellation {
9470            selection_group_id: group_id,
9471            member_key: selection_key_from_value(
9472                event.payload.get("member_key"),
9473                member_base_sequence,
9474            )?,
9475            member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9476            member_base_sequence,
9477            member_size: required_selection_usize(&event.payload, "member_size")?,
9478            operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9479                invalid_recorded_history(
9480                    "selection_cancellation_invalid",
9481                    member_base_sequence,
9482                    "selection operation kind",
9483                    &event.payload.to_string(),
9484                    "selection cancellation is missing its operation kind",
9485                )
9486            })?,
9487            operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9488                || {
9489                    invalid_recorded_history(
9490                        "selection_cancellation_invalid",
9491                        member_base_sequence,
9492                        "selection operation identity",
9493                        &event.payload.to_string(),
9494                        "selection cancellation is missing its operation identity",
9495                    )
9496                },
9497            )?,
9498        };
9499        if let Some(existing) = cancelled.iter().find(|recorded| {
9500            recorded.selection_group_id == marker.selection_group_id
9501                && recorded.member_base_sequence == marker.member_base_sequence
9502        }) {
9503            if existing != &marker {
9504                return Err(invalid_recorded_history(
9505                    "selection_cancellation_conflict",
9506                    member_base_sequence,
9507                    "one stable SelectionOperationCancelled marker",
9508                    &event.payload.to_string(),
9509                    "selection cancellation history contains conflicting member metadata",
9510                ));
9511            }
9512        } else {
9513            cancelled.push(marker);
9514        }
9515    }
9516    Ok(cancelled)
9517}
9518
9519fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9520    payload
9521        .get(field)
9522        .and_then(value_as_u64)
9523        .filter(|value| *value > 0)
9524        .ok_or_else(|| {
9525            invalid_recorded_history(
9526                "selection_marker_invalid",
9527                0,
9528                &format!("positive integer {field}"),
9529                &payload.to_string(),
9530                "selection history contains invalid durable identity metadata",
9531            )
9532        })
9533}
9534
9535fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9536    required_selection_usize_allow_zero(payload, field).and_then(|value| {
9537        if value > 0 {
9538            Ok(value)
9539        } else {
9540            Err(invalid_recorded_history(
9541                "selection_marker_invalid",
9542                0,
9543                &format!("positive integer {field}"),
9544                &payload.to_string(),
9545                "selection history contains invalid durable identity metadata",
9546            ))
9547        }
9548    })
9549}
9550
9551fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9552    payload
9553        .get(field)
9554        .and_then(value_as_u64)
9555        .and_then(|value| usize::try_from(value).ok())
9556        .ok_or_else(|| {
9557            invalid_recorded_history(
9558                "selection_marker_invalid",
9559                0,
9560                &format!("non-negative integer {field}"),
9561                &payload.to_string(),
9562                "selection history contains invalid durable identity metadata",
9563            )
9564        })
9565}
9566
9567#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9568struct RecordedActivityOptions {
9569    task_queue: RecordedSnapshotValue<Option<String>>,
9570    execution_mode: RecordedSnapshotValue<Option<String>>,
9571    retry_policy: ActivityRetrySnapshot,
9572}
9573
9574#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9575enum RecordedSnapshotValue<T> {
9576    /// Older history did not persist this field, so it cannot constrain replay.
9577    Unknown,
9578    Known(T),
9579}
9580
9581impl<T: PartialEq> RecordedSnapshotValue<T> {
9582    fn matches_current(&self, current: &Self) -> bool {
9583        match self {
9584            Self::Unknown => true,
9585            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9586        }
9587    }
9588}
9589
9590#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9591struct ActivityRetrySnapshot {
9592    snapshot_version: RecordedSnapshotValue<Option<u64>>,
9593    max_attempts: RecordedSnapshotValue<Option<u64>>,
9594    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9595    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9596    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9597    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9598    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9599    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9600}
9601
9602impl ActivityRetrySnapshot {
9603    fn matches_current(&self, current: &Self) -> bool {
9604        self.snapshot_version
9605            .matches_current(&current.snapshot_version)
9606            && self.max_attempts.matches_current(&current.max_attempts)
9607            && self
9608                .backoff_seconds
9609                .matches_current(&current.backoff_seconds)
9610            && self
9611                .start_to_close_timeout
9612                .matches_current(&current.start_to_close_timeout)
9613            && self
9614                .schedule_to_start_timeout
9615                .matches_current(&current.schedule_to_start_timeout)
9616            && self
9617                .schedule_to_close_timeout
9618                .matches_current(&current.schedule_to_close_timeout)
9619            && self
9620                .heartbeat_timeout
9621                .matches_current(&current.heartbeat_timeout)
9622            && self
9623                .non_retryable_error_types
9624                .matches_current(&current.non_retryable_error_types)
9625    }
9626}
9627
9628fn recorded_optional_u64(
9629    object: Option<&serde_json::Map<String, Value>>,
9630    field: &str,
9631) -> RecordedSnapshotValue<Option<u64>> {
9632    match object.and_then(|object| object.get(field)) {
9633        None => RecordedSnapshotValue::Unknown,
9634        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9635        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9636    }
9637}
9638
9639fn recorded_optional_string(
9640    object: &serde_json::Map<String, Value>,
9641    field: &str,
9642) -> RecordedSnapshotValue<Option<String>> {
9643    match object.get(field) {
9644        None => RecordedSnapshotValue::Unknown,
9645        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9646        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9647    }
9648}
9649
9650fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9651    let policy = policy.and_then(Value::as_object);
9652    let backoff_seconds = policy
9653        .and_then(|policy| policy.get("backoff_seconds"))
9654        .and_then(Value::as_array)
9655        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9656        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9657    let mut non_retryable_error_types = Vec::new();
9658    for error_type in policy
9659        .and_then(|policy| policy.get("non_retryable_error_types"))
9660        .and_then(Value::as_array)
9661        .into_iter()
9662        .flatten()
9663        .filter_map(Value::as_str)
9664        .map(str::trim)
9665        .filter(|error_type| !error_type.is_empty())
9666    {
9667        if !non_retryable_error_types
9668            .iter()
9669            .any(|recorded| recorded == error_type)
9670        {
9671            non_retryable_error_types.push(error_type.to_string());
9672        }
9673    }
9674
9675    ActivityRetrySnapshot {
9676        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9677        max_attempts: recorded_optional_u64(policy, "max_attempts"),
9678        backoff_seconds,
9679        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9680        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9681        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9682        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9683        non_retryable_error_types: if policy
9684            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9685        {
9686            RecordedSnapshotValue::Known(non_retryable_error_types)
9687        } else {
9688            RecordedSnapshotValue::Unknown
9689        },
9690    }
9691}
9692
9693fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9694    let policy = options.retry_policy.as_ref();
9695    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9696        Some(Value::Null) => None,
9697        Some(value) => value_as_u64(value),
9698        None => Some(1),
9699    };
9700    let backoff_seconds = policy
9701        .and_then(|policy| policy.get("backoff_seconds"))
9702        .and_then(Value::as_array)
9703        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9704        .unwrap_or_default();
9705    let non_retryable_error_types = policy
9706        .and_then(|policy| policy.get("non_retryable_error_types"))
9707        .and_then(Value::as_array)
9708        .into_iter()
9709        .flatten()
9710        .filter_map(Value::as_str)
9711        .map(str::to_string)
9712        .collect();
9713
9714    ActivityRetrySnapshot {
9715        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9716        max_attempts: RecordedSnapshotValue::Known(max_attempts),
9717        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9718        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9719        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9720        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9721        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9722        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9723    }
9724}
9725
9726fn activity_options_description(options: &RecordedActivityOptions) -> String {
9727    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9728}
9729
9730impl RecordedCommand {
9731    fn sequence(&self) -> u64 {
9732        match self {
9733            Self::Activity { sequence, .. }
9734            | Self::Timer { sequence, .. }
9735            | Self::ChildWorkflow { sequence, .. }
9736            | Self::SignalWait { sequence, .. }
9737            | Self::ConditionWait { sequence, .. }
9738            | Self::SearchAttributes { sequence, .. }
9739            | Self::SideEffect { sequence, .. }
9740            | Self::VersionMarker { sequence, .. }
9741            | Self::Memo { sequence, .. } => *sequence,
9742        }
9743    }
9744
9745    fn shape(&self) -> &'static str {
9746        match self {
9747            Self::Activity { .. } => "activity",
9748            Self::Timer { .. } => "timer",
9749            Self::ChildWorkflow { .. } => "child workflow",
9750            Self::SignalWait { .. } => "signal wait",
9751            Self::ConditionWait { .. } => "condition wait",
9752            Self::SearchAttributes { .. } => "search-attribute update",
9753            Self::SideEffect { .. } => "side effect",
9754            Self::VersionMarker { .. } => "version marker",
9755            Self::Memo { .. } => "memo upsert",
9756        }
9757    }
9758}
9759
9760fn ensure_version_supported(
9761    change_id: &str,
9762    version: i32,
9763    min_supported: i32,
9764    max_supported: i32,
9765    sequence: u64,
9766) -> Result<()> {
9767    if (min_supported..=max_supported).contains(&version) {
9768        return Ok(());
9769    }
9770    Err(Error::NonDeterministicReplay(ReplayFailure::new(
9771        "version_marker_incompatible_range",
9772        (sequence != 0).then_some(sequence),
9773        Some(format!("{min_supported}..={max_supported}")),
9774        Some(format!("{change_id}:{version}")),
9775        "recorded workflow version is outside the range supported by current code",
9776    )))
9777}
9778
9779#[derive(Clone, Debug)]
9780struct ResumeSignal {
9781    signal_name: String,
9782    arguments: Vec<AvroValue>,
9783}
9784
9785const MAX_PARALLEL_OPERATIONS: usize = 1000;
9786
9787fn parallel_group_prefix(kind: &str) -> &'static str {
9788    match kind {
9789        "activity" => "parallel-activities",
9790        "child" => "parallel-children",
9791        "timer" => "parallel-timers",
9792        _ => "parallel-calls",
9793    }
9794}
9795
9796fn parallel_group_entry(
9797    base_sequence: u64,
9798    size: usize,
9799    index: usize,
9800    kind: &str,
9801) -> ParallelGroupMetadata {
9802    ParallelGroupMetadata {
9803        parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9804        parallel_group_kind: kind.to_string(),
9805        parallel_group_base_sequence: base_sequence,
9806        parallel_group_size: size,
9807        parallel_group_index: index,
9808        parallel_group_mode: None,
9809        selection_member_key: None,
9810        selection_member_index: None,
9811        selection_member_base_sequence: None,
9812        selection_member_size: None,
9813        selection_member_kind: None,
9814    }
9815}
9816
9817struct SelectionMemberMetadata {
9818    key: SelectionKey,
9819    index: usize,
9820    base_sequence: u64,
9821    size: usize,
9822    kind: String,
9823}
9824
9825fn selection_group_entry(
9826    base_sequence: u64,
9827    size: usize,
9828    index: usize,
9829    kind: &str,
9830    member: &SelectionMemberMetadata,
9831) -> ParallelGroupMetadata {
9832    ParallelGroupMetadata {
9833        parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9834        parallel_group_kind: kind.to_string(),
9835        parallel_group_base_sequence: base_sequence,
9836        parallel_group_size: size,
9837        parallel_group_index: index,
9838        parallel_group_mode: Some("select".to_string()),
9839        selection_member_key: Some(member.key.clone()),
9840        selection_member_index: Some(member.index),
9841        selection_member_base_sequence: Some(member.base_sequence),
9842        selection_member_size: Some(member.size),
9843        selection_member_kind: Some(member.kind.clone()),
9844    }
9845}
9846
9847fn apply_parallel_group_path(
9848    command: &mut serde_json::Map<String, Value>,
9849    path: &[ParallelGroupMetadata],
9850) {
9851    let Some(inner) = path.last() else {
9852        return;
9853    };
9854    command.insert(
9855        "parallel_group_id".to_string(),
9856        json!(inner.parallel_group_id),
9857    );
9858    command.insert(
9859        "parallel_group_kind".to_string(),
9860        json!(inner.parallel_group_kind),
9861    );
9862    command.insert(
9863        "parallel_group_base_sequence".to_string(),
9864        json!(inner.parallel_group_base_sequence),
9865    );
9866    command.insert(
9867        "parallel_group_size".to_string(),
9868        json!(inner.parallel_group_size),
9869    );
9870    command.insert(
9871        "parallel_group_index".to_string(),
9872        json!(inner.parallel_group_index),
9873    );
9874    if let Some(mode) = &inner.parallel_group_mode {
9875        command.insert("parallel_group_mode".to_string(), json!(mode));
9876    }
9877    if let Some(key) = &inner.selection_member_key {
9878        command.insert("selection_member_key".to_string(), json!(key));
9879    }
9880    if let Some(index) = inner.selection_member_index {
9881        command.insert("selection_member_index".to_string(), json!(index));
9882    }
9883    if let Some(base_sequence) = inner.selection_member_base_sequence {
9884        command.insert(
9885            "selection_member_base_sequence".to_string(),
9886            json!(base_sequence),
9887        );
9888    }
9889    if let Some(size) = inner.selection_member_size {
9890        command.insert("selection_member_size".to_string(), json!(size));
9891    }
9892    if let Some(kind) = &inner.selection_member_kind {
9893        command.insert("selection_member_kind".to_string(), json!(kind));
9894    }
9895    command.insert("parallel_group_path".to_string(), json!(path));
9896}
9897
9898fn ensure_parallel_path_matches(
9899    sequence: u64,
9900    recorded: Option<&[ParallelGroupMetadata]>,
9901    expected: &[ParallelGroupMetadata],
9902) -> Result<()> {
9903    match (recorded, expected.is_empty()) {
9904        (None, true) => Ok(()),
9905        (Some(recorded), false) if recorded == expected => Ok(()),
9906        (None, false) => Err(invalid_recorded_history(
9907            "parallel_group_metadata_missing",
9908            sequence,
9909            &serde_json::to_string(expected).unwrap_or_default(),
9910            "<missing>",
9911            "recorded parallel member is missing its durable group path",
9912        )),
9913        (Some(recorded), true) => Err(invalid_recorded_history(
9914            "parallel_group_shape_mismatch",
9915            sequence,
9916            "sequential command",
9917            &serde_json::to_string(recorded).unwrap_or_default(),
9918            "recorded command belonged to a parallel group but current code schedules it sequentially",
9919        )),
9920        (Some(recorded), false) => Err(invalid_recorded_history(
9921            "parallel_group_shape_mismatch",
9922            sequence,
9923            &serde_json::to_string(recorded).unwrap_or_default(),
9924            &serde_json::to_string(expected).unwrap_or_default(),
9925            "recorded parallel-group identity or path changed during replay",
9926        )),
9927    }
9928}
9929
9930#[derive(Clone, Debug)]
9931enum ParallelShape {
9932    Leaf,
9933    Group(Vec<ParallelShape>),
9934}
9935
9936struct ParallelDescriptor {
9937    operation: ParallelOperation,
9938    offset: usize,
9939    member_path: Vec<usize>,
9940    group_path: Vec<ParallelGroupMetadata>,
9941}
9942
9943fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9944    operations
9945        .iter()
9946        .map(|operation| match operation {
9947            ParallelOperation::Group(children) => parallel_leaf_count(children),
9948            _ => 1,
9949        })
9950        .sum()
9951}
9952
9953fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
9954    match operation {
9955        ParallelOperation::Activity { .. } => Some("activity"),
9956        ParallelOperation::ChildWorkflow { .. } => Some("child"),
9957        ParallelOperation::Timer(_) => Some("timer"),
9958        ParallelOperation::Signal(_) => Some("signal"),
9959        ParallelOperation::Condition { .. } => Some("condition"),
9960        ParallelOperation::Group(children) => parallel_group_kind(children),
9961    }
9962}
9963
9964fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
9965    let mut kind = None;
9966    for operation in operations {
9967        let Some(operation_kind) = parallel_operation_kind(operation) else {
9968            continue;
9969        };
9970        match kind {
9971            None => kind = Some(operation_kind),
9972            Some(current) if current == operation_kind => {}
9973            Some(_) => return Some("mixed"),
9974        }
9975    }
9976    kind
9977}
9978
9979fn validate_parallel_operations(
9980    operations: &[ParallelOperation],
9981    member_path: &mut Vec<usize>,
9982    root: bool,
9983) -> Result<()> {
9984    let leaves = parallel_leaf_count(operations);
9985    if leaves > MAX_PARALLEL_OPERATIONS {
9986        return Err(Error::InvalidParallelGroup(ParallelGroupError {
9987            reason: "fan_out_limit_exceeded",
9988            member_path: member_path.clone(),
9989            message: format!(
9990                "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
9991            ),
9992        }));
9993    }
9994    if !root && operations.is_empty() {
9995        return Err(Error::InvalidParallelGroup(ParallelGroupError {
9996            reason: "nested_group_empty",
9997            member_path: member_path.clone(),
9998            message: "a nested group must contain at least one durable leaf".to_string(),
9999        }));
10000    }
10001
10002    for (index, operation) in operations.iter().enumerate() {
10003        member_path.push(index);
10004        match operation {
10005            ParallelOperation::Activity {
10006                options, arguments, ..
10007            } => {
10008                options
10009                    .validate()
10010                    .map_err(|error| Error::InvalidActivityOptions(error))?;
10011                if let Err(error) = arguments {
10012                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10013                        reason: "arguments_invalid",
10014                        member_path: member_path.clone(),
10015                        message: error.to_string(),
10016                    }));
10017                }
10018            }
10019            ParallelOperation::ChildWorkflow {
10020                options, arguments, ..
10021            } => {
10022                validate_parallel_child_options(options)?;
10023                if let Err(error) = arguments {
10024                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10025                        reason: "arguments_invalid",
10026                        member_path: member_path.clone(),
10027                        message: error.to_string(),
10028                    }));
10029                }
10030            }
10031            ParallelOperation::Timer(duration)
10032                if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10033            {
10034                return Err(Error::TimerDurationOverflow);
10035            }
10036            ParallelOperation::Timer(_) => {}
10037            ParallelOperation::Signal(signal_name) => {
10038                validate_user_signal_name(signal_name)?;
10039                if signal_name.trim().is_empty() {
10040                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10041                        reason: "signal_name_empty",
10042                        member_path: member_path.clone(),
10043                        message: "signal wait name must not be empty".to_string(),
10044                    }));
10045                }
10046            }
10047            ParallelOperation::Condition { options, .. } => {
10048                options.validate()?;
10049            }
10050            ParallelOperation::Group(children) => {
10051                validate_parallel_operations(children, member_path, false)?;
10052            }
10053        }
10054        member_path.pop();
10055    }
10056    Ok(())
10057}
10058
10059fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10060    if options.task_queue.trim().is_empty() {
10061        return Err(Error::InvalidChildWorkflowOptions(
10062            "task_queue must not be empty".to_string(),
10063        ));
10064    }
10065    for (name, value) in [
10066        (
10067            "execution_timeout_seconds",
10068            options.execution_timeout_seconds,
10069        ),
10070        ("run_timeout_seconds", options.run_timeout_seconds),
10071    ] {
10072        if value == Some(0) {
10073            return Err(Error::InvalidChildWorkflowOptions(format!(
10074                "{name} must be at least 1"
10075            )));
10076        }
10077    }
10078    if options
10079        .retry_policy
10080        .as_ref()
10081        .is_some_and(|policy| policy.max_attempts == Some(0))
10082    {
10083        return Err(Error::InvalidChildWorkflowOptions(
10084            "retry_policy.max_attempts must be at least 1".to_string(),
10085        ));
10086    }
10087    Ok(())
10088}
10089
10090fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10091    ParallelShape::Group(
10092        operations
10093            .iter()
10094            .map(|operation| match operation {
10095                ParallelOperation::Group(children) => parallel_shape(children),
10096                _ => ParallelShape::Leaf,
10097            })
10098            .collect(),
10099    )
10100}
10101
10102fn parallel_descriptors(
10103    operations: Vec<ParallelOperation>,
10104    base_sequence: u64,
10105) -> Result<Vec<ParallelDescriptor>> {
10106    let size = parallel_leaf_count(&operations);
10107    let kind = parallel_group_kind(&operations).unwrap_or("activity");
10108    let mut descriptors = Vec::with_capacity(size);
10109    let mut cursor = 0;
10110
10111    for (index, operation) in operations.into_iter().enumerate() {
10112        match operation {
10113            ParallelOperation::Group(children) => {
10114                let child_base = base_sequence
10115                    .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10116                    .ok_or(Error::TimerDurationOverflow)?;
10117                for mut descriptor in parallel_descriptors(children, child_base)? {
10118                    let outer_index = cursor + descriptor.offset;
10119                    descriptor.group_path.insert(
10120                        0,
10121                        parallel_group_entry(base_sequence, size, outer_index, kind),
10122                    );
10123                    descriptor.member_path.insert(0, index);
10124                    descriptor.offset = outer_index;
10125                    descriptors.push(descriptor);
10126                }
10127                cursor = descriptors.len();
10128            }
10129            operation => {
10130                descriptors.push(ParallelDescriptor {
10131                    operation,
10132                    offset: cursor,
10133                    member_path: vec![index],
10134                    group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10135                });
10136                cursor += 1;
10137            }
10138        }
10139    }
10140    Ok(descriptors)
10141}
10142
10143enum ParallelLeafCall {
10144    Activity(ActivityCall),
10145    ChildWorkflow(ChildWorkflowCall),
10146    Timer(TimerCall),
10147    Signal(SignalCall),
10148    Condition(ConditionWaitCall),
10149}
10150
10151fn parallel_leaf_call(
10152    ctx: &WorkflowContext,
10153    operation: ParallelOperation,
10154    parallel_group_path: Vec<ParallelGroupMetadata>,
10155) -> ParallelLeafCall {
10156    match operation {
10157        ParallelOperation::Activity {
10158            activity_type,
10159            options,
10160            arguments,
10161        } => ParallelLeafCall::Activity(ActivityCall {
10162            ctx: ctx.clone(),
10163            activity_type,
10164            options,
10165            args: Some(arguments),
10166            scheduled: false,
10167            parallel_group_path,
10168        }),
10169        ParallelOperation::ChildWorkflow {
10170            workflow_type,
10171            options,
10172            arguments,
10173        } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10174            ctx: ctx.clone(),
10175            workflow_type,
10176            options,
10177            args: Some(arguments),
10178            scheduled: false,
10179            matched_pending: false,
10180            parallel_group_path,
10181        }),
10182        ParallelOperation::Timer(duration) => {
10183            let delay_seconds = duration
10184                .as_secs()
10185                .checked_add(u64::from(duration.subsec_nanos() > 0));
10186            ParallelLeafCall::Timer(TimerCall {
10187                ctx: ctx.clone(),
10188                delay_seconds,
10189                scheduled: false,
10190                matched_pending: false,
10191                parallel_group_path,
10192            })
10193        }
10194        ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10195            ctx: ctx.clone(),
10196            signal_name,
10197            runtime_reserved_allowed: false,
10198            opened_wait: false,
10199            matched_pending: false,
10200            parallel_group_path,
10201        }),
10202        ParallelOperation::Condition { options, predicate } => {
10203            ParallelLeafCall::Condition(ConditionWaitCall {
10204                ctx: ctx.clone(),
10205                options,
10206                predicate,
10207                occurrence_id: None,
10208                opened_wait: false,
10209                parallel_group_path,
10210            })
10211        }
10212        ParallelOperation::Group(_) => {
10213            unreachable!("parallel descriptors contain only durable leaves")
10214        }
10215    }
10216}
10217
10218impl ParallelLeafCall {
10219    fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10220        match self {
10221            Self::Activity(call) => Pin::new(call)
10222                .poll_avro_value(cx)
10223                .map_ok(ParallelAvroResult::Activity),
10224            Self::ChildWorkflow(call) => Pin::new(call)
10225                .poll_avro_value(cx)
10226                .map_ok(ParallelAvroResult::ChildWorkflow),
10227            Self::Timer(call) => Pin::new(call)
10228                .poll(cx)
10229                .map_ok(|()| ParallelAvroResult::Timer),
10230            Self::Signal(call) => Pin::new(call)
10231                .poll_avro_value(cx)
10232                .map_ok(ParallelAvroResult::Signal),
10233            Self::Condition(call) => Pin::new(call)
10234                .poll(cx)
10235                .map_ok(ParallelAvroResult::Condition),
10236        }
10237    }
10238}
10239
10240struct ParallelLeaf {
10241    call: ParallelLeafCall,
10242    member_path: Vec<usize>,
10243    group_path: Vec<ParallelGroupMetadata>,
10244    result: Option<ParallelAvroResult>,
10245}
10246
10247/// Future returned by [`WorkflowContext::parallel`].
10248pub struct ParallelCall {
10249    ctx: WorkflowContext,
10250    operations: Option<Vec<ParallelOperation>>,
10251    shape: Option<ParallelShape>,
10252    leaves: Vec<ParallelLeaf>,
10253}
10254
10255impl ParallelCall {
10256    fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10257        Self {
10258            ctx,
10259            operations: Some(operations),
10260            shape: None,
10261            leaves: Vec::new(),
10262        }
10263    }
10264
10265    fn initialize(&mut self) -> Result<()> {
10266        let operations = self.operations.take().unwrap_or_default();
10267        validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10268        self.shape = Some(parallel_shape(&operations));
10269        if operations.is_empty() {
10270            return Ok(());
10271        }
10272
10273        let base_sequence = {
10274            let state = self
10275                .ctx
10276                .state
10277                .lock()
10278                .map_err(|_| Error::WorkflowStatePoisoned)?;
10279            if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10280                recorded.sequence()
10281            } else {
10282                let last = state
10283                    .recorded_commands
10284                    .last()
10285                    .map(RecordedCommand::sequence)
10286                    .unwrap_or(0);
10287                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10288                    .and_then(|sequence| sequence.checked_add(1))
10289                    .ok_or_else(|| {
10290                        Error::InvalidParallelGroup(ParallelGroupError {
10291                            reason: "sequence_overflow",
10292                            member_path: Vec::new(),
10293                            message: "parallel group sequence identity overflowed u64".to_string(),
10294                        })
10295                    })?
10296            }
10297        };
10298
10299        self.leaves = parallel_descriptors(operations, base_sequence)?
10300            .into_iter()
10301            .map(|descriptor| {
10302                let call = parallel_leaf_call(
10303                    &self.ctx,
10304                    descriptor.operation,
10305                    descriptor.group_path.clone(),
10306                );
10307                ParallelLeaf {
10308                    call,
10309                    member_path: descriptor.member_path,
10310                    group_path: descriptor.group_path,
10311                    result: None,
10312                }
10313            })
10314            .collect();
10315        Ok(())
10316    }
10317
10318    fn poll_avro_value(
10319        mut self: Pin<&mut Self>,
10320        cx: &mut TaskContext<'_>,
10321    ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10322        if self.operations.is_some() {
10323            if let Err(error) = self.initialize() {
10324                return Poll::Ready(Err(error));
10325            }
10326        }
10327        if self.leaves.is_empty() {
10328            return Poll::Ready(Ok(Vec::new()));
10329        }
10330
10331        let mut failures = Vec::new();
10332        let mut pending = false;
10333        for (index, leaf) in self.leaves.iter_mut().enumerate() {
10334            if leaf.result.is_some() {
10335                continue;
10336            }
10337            match leaf.call.poll_avro_value(cx) {
10338                Poll::Ready(Ok(result)) => leaf.result = Some(result),
10339                Poll::Ready(Err(error)) => failures.push((index, error)),
10340                Poll::Pending => pending = true,
10341            }
10342        }
10343
10344        if !failures.is_empty() {
10345            if let Some(position) = failures
10346                .iter()
10347                .position(|(_, error)| workflow_task_integrity_error(error))
10348            {
10349                return Poll::Ready(Err(failures.remove(position).1));
10350            }
10351            failures.sort_by_key(|(index, _)| *index);
10352            let (failed_index, cause) = failures.remove(0);
10353            let failed = &self.leaves[failed_index];
10354            let completed = self
10355                .leaves
10356                .iter()
10357                .filter_map(|leaf| {
10358                    leaf.result
10359                        .clone()
10360                        .and_then(|result| result.into_json_result().ok())
10361                        .map(|result| ParallelCompletion {
10362                            member_path: leaf.member_path.clone(),
10363                            result,
10364                        })
10365                })
10366                .collect();
10367            let group_id = failed
10368                .group_path
10369                .first()
10370                .map(|entry| entry.parallel_group_id.clone())
10371                .unwrap_or_default();
10372            return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10373                group_id,
10374                member_path: failed.member_path.clone(),
10375                group_path: failed.group_path.clone(),
10376                completed,
10377                cause: Box::new(cause),
10378            })));
10379        }
10380        if pending {
10381            return Poll::Pending;
10382        }
10383
10384        let mut flat_results = self
10385            .leaves
10386            .iter_mut()
10387            .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10388            .collect::<Vec<_>>()
10389            .into_iter();
10390        let results = parallel_results_for_shape(
10391            self.shape.as_ref().expect("initialized parallel shape"),
10392            &mut flat_results,
10393        );
10394        Poll::Ready(Ok(match results {
10395            ParallelAvroResult::Group(results) => results,
10396            ParallelAvroResult::Activity(_)
10397            | ParallelAvroResult::ChildWorkflow(_)
10398            | ParallelAvroResult::Timer
10399            | ParallelAvroResult::Signal(_)
10400            | ParallelAvroResult::Condition(_) => {
10401                unreachable!("root parallel shape is a group")
10402            }
10403        }))
10404    }
10405}
10406
10407fn parallel_results_for_shape(
10408    shape: &ParallelShape,
10409    flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10410) -> ParallelAvroResult {
10411    match shape {
10412        ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10413        ParallelShape::Group(children) => ParallelAvroResult::Group(
10414            children
10415                .iter()
10416                .map(|child| parallel_results_for_shape(child, flat_results))
10417                .collect(),
10418        ),
10419    }
10420}
10421
10422impl Future for ParallelCall {
10423    type Output = Result<Vec<ParallelResult>>;
10424
10425    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10426        self.poll_avro_value(cx)
10427            .map_ok(|results| {
10428                results
10429                    .into_iter()
10430                    .map(ParallelAvroResult::into_json_result)
10431                    .collect::<Result<Vec<_>>>()
10432            })
10433            .map_ok(|result| result)
10434            .flatten_result()
10435    }
10436}
10437
10438#[derive(Clone, Debug)]
10439struct SelectionMemberPlan {
10440    key: SelectionKey,
10441    index: usize,
10442    base_sequence: u64,
10443    size: usize,
10444    kind: String,
10445    shape: ParallelShape,
10446    leaf_start: usize,
10447}
10448
10449fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10450    match operation {
10451        ParallelOperation::Activity { .. } => "activity",
10452        ParallelOperation::ChildWorkflow { .. } => "child",
10453        ParallelOperation::Timer(_) => "timer",
10454        ParallelOperation::Signal(_) => "signal",
10455        ParallelOperation::Condition { .. } => "condition",
10456        ParallelOperation::Group(_) => "group",
10457    }
10458}
10459
10460fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10461    match operation {
10462        ParallelOperation::Group(children) => parallel_shape(children),
10463        _ => ParallelShape::Leaf,
10464    }
10465}
10466
10467fn selection_descriptors(
10468    operations: Vec<(SelectionKey, ParallelOperation)>,
10469    base_sequence: u64,
10470) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10471    if operations.is_empty() {
10472        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10473            reason: "selection_empty",
10474            member_path: Vec::new(),
10475            message: "durable selection requires at least one operation".to_string(),
10476        }));
10477    }
10478    let operation_refs = operations
10479        .iter()
10480        .map(|(_, operation)| operation)
10481        .collect::<Vec<_>>();
10482    let total_size = operation_refs
10483        .iter()
10484        .map(|operation| match operation {
10485            ParallelOperation::Group(children) => parallel_leaf_count(children),
10486            _ => 1,
10487        })
10488        .sum::<usize>();
10489    if total_size > MAX_PARALLEL_OPERATIONS {
10490        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10491            reason: "fan_out_limit_exceeded",
10492            member_path: Vec::new(),
10493            message: format!(
10494                "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10495            ),
10496        }));
10497    }
10498    let group_kind = {
10499        let mut kind = None;
10500        for operation in &operation_refs {
10501            let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10502            match kind {
10503                None => kind = Some(operation_kind),
10504                Some(current) if current == operation_kind => {}
10505                Some(_) => {
10506                    kind = Some("mixed");
10507                    break;
10508                }
10509            }
10510        }
10511        kind.unwrap_or("mixed")
10512    };
10513
10514    let mut descriptors = Vec::with_capacity(total_size);
10515    let mut members = Vec::with_capacity(operations.len());
10516    let mut cursor = 0usize;
10517    let mut seen_keys: Vec<SelectionKey> = Vec::new();
10518    for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10519        if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10520            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10521                reason: "selection_key_invalid",
10522                member_path: vec![member_index],
10523                message: "selection member keys must be non-empty strings or non-negative integers"
10524                    .to_string(),
10525            }));
10526        }
10527        if seen_keys.contains(&key) {
10528            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10529                reason: "selection_key_duplicate",
10530                member_path: vec![member_index],
10531                message: format!("selection member key {key:?} is duplicated"),
10532            }));
10533        }
10534        seen_keys.push(key.clone());
10535        let member_size = match &operation {
10536            ParallelOperation::Group(children) => parallel_leaf_count(children),
10537            _ => 1,
10538        };
10539        if member_size == 0 {
10540            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10541                reason: "selection_member_empty",
10542                member_path: vec![member_index],
10543                message: "a selection member must contain at least one durable leaf".to_string(),
10544            }));
10545        }
10546        let member_base = base_sequence
10547            .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10548            .ok_or(Error::TimerDurationOverflow)?;
10549        let member_kind = selection_operation_kind(&operation).to_string();
10550        let member_shape = selection_operation_shape(&operation);
10551        let leaf_start = descriptors.len();
10552        match operation {
10553            ParallelOperation::Group(children) => {
10554                validate_parallel_operations(&children, &mut vec![member_index], false)?;
10555                for mut descriptor in parallel_descriptors(children, member_base)? {
10556                    let flat_index = cursor + descriptor.offset;
10557                    descriptor.group_path.insert(
10558                        0,
10559                        selection_group_entry(
10560                            base_sequence,
10561                            total_size,
10562                            flat_index,
10563                            group_kind,
10564                            &SelectionMemberMetadata {
10565                                key: key.clone(),
10566                                index: member_index,
10567                                base_sequence: member_base,
10568                                size: member_size,
10569                                kind: member_kind.clone(),
10570                            },
10571                        ),
10572                    );
10573                    descriptor.member_path.insert(0, member_index);
10574                    descriptor.offset = flat_index;
10575                    descriptors.push(descriptor);
10576                }
10577            }
10578            operation => {
10579                validate_parallel_operations(
10580                    std::slice::from_ref(&operation),
10581                    &mut Vec::new(),
10582                    true,
10583                )?;
10584                descriptors.push(ParallelDescriptor {
10585                    operation,
10586                    offset: cursor,
10587                    member_path: vec![member_index],
10588                    group_path: vec![selection_group_entry(
10589                        base_sequence,
10590                        total_size,
10591                        cursor,
10592                        group_kind,
10593                        &SelectionMemberMetadata {
10594                            key: key.clone(),
10595                            index: member_index,
10596                            base_sequence: member_base,
10597                            size: member_size,
10598                            kind: member_kind.clone(),
10599                        },
10600                    )],
10601                });
10602            }
10603        }
10604        members.push(SelectionMemberPlan {
10605            key,
10606            index: member_index,
10607            base_sequence: member_base,
10608            size: member_size,
10609            kind: member_kind,
10610            shape: member_shape,
10611            leaf_start,
10612        });
10613        cursor += member_size;
10614    }
10615    Ok((descriptors, members))
10616}
10617
10618struct SelectionLeaf {
10619    call: ParallelLeafCall,
10620    outcome: Option<Result<ParallelAvroResult>>,
10621}
10622
10623/// Stable reference to one member of a durable selection group.
10624#[derive(Clone)]
10625pub struct DurableOperationHandle {
10626    ctx: WorkflowContext,
10627    pub key: SelectionKey,
10628    pub index: usize,
10629    pub kind: String,
10630    pub identity: String,
10631    pub base_sequence: u64,
10632    pub size: usize,
10633    pub selection_group_id: String,
10634    shape: ParallelShape,
10635}
10636
10637impl std::fmt::Debug for DurableOperationHandle {
10638    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10639        formatter
10640            .debug_struct("DurableOperationHandle")
10641            .field("key", &self.key)
10642            .field("index", &self.index)
10643            .field("kind", &self.kind)
10644            .field("identity", &self.identity)
10645            .field("base_sequence", &self.base_sequence)
10646            .field("size", &self.size)
10647            .field("selection_group_id", &self.selection_group_id)
10648            .finish()
10649    }
10650}
10651
10652impl DurableOperationHandle {
10653    /// Await this member after another member has already won the selection.
10654    pub fn await_result(&self) -> DurableOperationAwaitCall {
10655        DurableOperationAwaitCall {
10656            handle: self.clone(),
10657        }
10658    }
10659
10660    /// Request cancellation without affecting siblings. The future resolves
10661    /// to unit; only `SelectionOperationCancelled` history proves cancellation
10662    /// beat a concurrently committed terminal result.
10663    pub fn cancel(&self) -> CancelDurableOperationCall {
10664        CancelDurableOperationCall {
10665            handle: self.clone(),
10666            emitted: false,
10667        }
10668    }
10669}
10670
10671/// The one winner committed for a durable selection group.
10672#[derive(Debug)]
10673pub struct SelectionResult {
10674    pub key: SelectionKey,
10675    pub index: usize,
10676    pub kind: String,
10677    pub identity: String,
10678    pub value: Option<ParallelResult>,
10679    pub failure: Option<Error>,
10680    pub winner: DurableOperationHandle,
10681    pub handles: Vec<DurableOperationHandle>,
10682}
10683
10684impl SelectionResult {
10685    pub fn succeeded(&self) -> bool {
10686        self.failure.is_none()
10687    }
10688
10689    pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10690        self.handles.iter().find(|handle| &handle.key == key)
10691    }
10692
10693    pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10694        self.handles
10695            .iter()
10696            .filter(|handle| handle.index != self.index)
10697            .collect()
10698    }
10699
10700    pub fn into_result(self) -> Result<ParallelResult> {
10701        match (self.value, self.failure) {
10702            (Some(value), None) => Ok(value),
10703            (_, Some(error)) => Err(error),
10704            _ => Err(Error::WorkerLoop(
10705                "selection result contained neither a value nor a failure".to_string(),
10706            )),
10707        }
10708    }
10709}
10710
10711/// Future returned by [`WorkflowContext::select`].
10712pub struct SelectCall {
10713    ctx: WorkflowContext,
10714    operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10715    members: Vec<SelectionMemberPlan>,
10716    leaves: Vec<SelectionLeaf>,
10717    group_id: Option<String>,
10718}
10719
10720impl SelectCall {
10721    fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10722        Self {
10723            ctx,
10724            operations: Some(operations),
10725            members: Vec::new(),
10726            leaves: Vec::new(),
10727            group_id: None,
10728        }
10729    }
10730
10731    fn initialize(&mut self) -> Result<()> {
10732        let operations = self.operations.take().unwrap_or_default();
10733        let base_sequence = {
10734            let state = self
10735                .ctx
10736                .state
10737                .lock()
10738                .map_err(|_| Error::WorkflowStatePoisoned)?;
10739            if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10740                marker.selection_group_base_sequence
10741            } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10742                recorded.sequence()
10743            } else {
10744                let last = state
10745                    .recorded_commands
10746                    .last()
10747                    .map(RecordedCommand::sequence)
10748                    .unwrap_or(0);
10749                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10750                    .and_then(|sequence| sequence.checked_add(1))
10751                    .ok_or(Error::TimerDurationOverflow)?
10752            }
10753        };
10754        let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10755        let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10756        self.leaves = descriptors
10757            .into_iter()
10758            .map(|descriptor| SelectionLeaf {
10759                call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10760                outcome: None,
10761            })
10762            .collect();
10763        self.members = members;
10764        self.group_id = Some(group_id);
10765        Ok(())
10766    }
10767}
10768
10769impl Future for SelectCall {
10770    type Output = Result<SelectionResult>;
10771
10772    fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10773        if self.operations.is_some() {
10774            if let Err(error) = self.initialize() {
10775                return Poll::Ready(Err(error));
10776            }
10777        }
10778
10779        for leaf in &mut self.leaves {
10780            if leaf.outcome.is_some() {
10781                continue;
10782            }
10783            if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10784                if outcome
10785                    .as_ref()
10786                    .err()
10787                    .is_some_and(workflow_task_integrity_error)
10788                {
10789                    return Poll::Ready(outcome.map(|_| unreachable!()));
10790                }
10791                leaf.outcome = Some(outcome);
10792            }
10793        }
10794
10795        let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10796        let selection_member_range = self
10797            .members
10798            .first()
10799            .map(|member| member.base_sequence)
10800            .zip(self.leaves.len().try_into().ok())
10801            .map(|(base_sequence, size): (u64, u64)| {
10802                base_sequence..base_sequence.saturating_add(size)
10803            });
10804        let marker = {
10805            let mut state = match self.ctx.state.lock() {
10806                Ok(state) => state,
10807                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10808            };
10809            let marker = state
10810                .selection_markers
10811                .get(state.selection_marker_cursor)
10812                .cloned();
10813            if marker.is_none()
10814                && all_members_terminal
10815                && selection_member_range.as_ref().is_some_and(|member_range| {
10816                    state
10817                        .recorded_commands
10818                        .iter()
10819                        .any(|command| member_range.contains(&command.sequence()))
10820                })
10821            {
10822                // Terminal member history can be committed before the server's
10823                // SelectionResolved marker becomes visible to this task. That
10824                // marker is the durable barrier the selection is still waiting
10825                // on, so an otherwise commandless replay remains legitimately
10826                // pending instead of failing as an untracked yield.
10827                state.matched_recorded_pending = true;
10828            }
10829            marker
10830        };
10831        let Some(marker) = marker else {
10832            return Poll::Pending;
10833        };
10834        if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10835            || marker.selection_group_size != self.leaves.len()
10836            || self.members.first().map(|member| member.base_sequence)
10837                != Some(marker.selection_group_base_sequence)
10838        {
10839            return Poll::Ready(Err(invalid_recorded_history(
10840                "selection_group_shape_mismatch",
10841                marker.selection_group_base_sequence,
10842                self.group_id
10843                    .as_deref()
10844                    .unwrap_or("initialized selection group"),
10845                &marker.selection_group_id,
10846                "recorded selection group differs from current workflow code",
10847            )));
10848        }
10849        let Some(member_position) = self.members.iter().position(|member| {
10850            member.key == marker.member_key
10851                && member.index == marker.member_index
10852                && member.base_sequence == marker.member_base_sequence
10853                && member.size == marker.member_size
10854                && member.kind == marker.operation_kind
10855        }) else {
10856            return Poll::Ready(Err(invalid_recorded_history(
10857                "selection_member_shape_mismatch",
10858                marker.member_base_sequence,
10859                "winner member matching current workflow code",
10860                &format!("{:?}", marker.member_key),
10861                "recorded selection winner differs from the authored member identity",
10862            )));
10863        };
10864        let member = self.members[member_position].clone();
10865        let (handles, resolution_sequence) = {
10866            let mut state = match self.ctx.state.lock() {
10867                Ok(state) => state,
10868                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10869            };
10870            let identities = self
10871                .members
10872                .iter()
10873                .map(|candidate| {
10874                    selection_operation_identity(
10875                        &state,
10876                        &candidate.kind,
10877                        candidate.base_sequence,
10878                        candidate.size,
10879                    )
10880                })
10881                .collect::<Vec<_>>();
10882            if let Some((position, missing)) = identities
10883                .iter()
10884                .enumerate()
10885                .find(|(_, identity)| identity.is_empty())
10886                .map(|(position, identity)| (position, identity.clone()))
10887            {
10888                let candidate = &self.members[position];
10889                return Poll::Ready(Err(invalid_recorded_history(
10890                    "selection_operation_identity_missing",
10891                    candidate.base_sequence,
10892                    &format!(
10893                        "durable {} resource identity from scheduled/open history",
10894                        candidate.kind
10895                    ),
10896                    &missing,
10897                    "selection member history is missing its canonical durable identity",
10898                )));
10899            }
10900            let expected_winner_identity = &identities[member_position];
10901            let resolution_sequence = match validated_selection_resolution_sequence(
10902                &state,
10903                &marker,
10904                &member,
10905                expected_winner_identity,
10906            ) {
10907                Ok(sequence) => sequence,
10908                Err(error) => return Poll::Ready(Err(error)),
10909            };
10910            let handles = self
10911                .members
10912                .iter()
10913                .zip(identities)
10914                .map(|(member, identity)| DurableOperationHandle {
10915                    ctx: self.ctx.clone(),
10916                    key: member.key.clone(),
10917                    index: member.index,
10918                    kind: member.kind.clone(),
10919                    identity,
10920                    base_sequence: member.base_sequence,
10921                    size: member.size,
10922                    selection_group_id: marker.selection_group_id.clone(),
10923                    shape: member.shape.clone(),
10924                })
10925                .collect::<Vec<_>>();
10926            if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
10927                return Poll::Ready(Err(error));
10928            }
10929            state.selection_marker_cursor += 1;
10930            (handles, resolution_sequence)
10931        };
10932
10933        let mut winner_failure = None;
10934        let mut flat_results = Vec::with_capacity(member.size);
10935        if marker.outcome == "failed" {
10936            let resolution_offset = match resolution_sequence
10937                .checked_sub(member.base_sequence)
10938                .and_then(|offset| usize::try_from(offset).ok())
10939            {
10940                Some(offset) if offset < member.size => offset,
10941                _ => {
10942                    return Poll::Ready(Err(invalid_recorded_history(
10943                        "selection_resolution_event_mismatch",
10944                        member.base_sequence,
10945                        "failure event within selected member bounds",
10946                        &resolution_sequence.to_string(),
10947                        "selection failure event is outside the authored member",
10948                    )))
10949                }
10950            };
10951            let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
10952            match leaf.outcome.take() {
10953                Some(Err(error)) => winner_failure = Some(error),
10954                _ => {
10955                    return Poll::Ready(Err(invalid_recorded_history(
10956                        "selection_winner_outcome_mismatch",
10957                        member.base_sequence,
10958                        "exact failed terminal history referenced by SelectionResolved",
10959                        "missing or successful resolution event",
10960                        "selection winner marker disagrees with terminal operation history",
10961                    )))
10962                }
10963            }
10964        } else {
10965            for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
10966                match leaf.outcome.take() {
10967                    Some(Ok(result)) => flat_results.push(result),
10968                    Some(Err(_)) => {
10969                        return Poll::Ready(Err(invalid_recorded_history(
10970                            "selection_winner_outcome_mismatch",
10971                            member.base_sequence,
10972                            "fully completed nested selection member",
10973                            "failed durable leaf",
10974                            "completed selection winner contains a failed leaf",
10975                        )))
10976                    }
10977                    None => {
10978                        return Poll::Ready(Err(invalid_recorded_history(
10979                            "selection_winner_unresolved",
10980                            member.base_sequence,
10981                            "terminal history for every completed winner leaf",
10982                            "pending member history",
10983                            "completed SelectionResolved member has an unfinished durable barrier",
10984                        )))
10985                    }
10986                }
10987            }
10988        }
10989        let value = if winner_failure.is_none() {
10990            let mut flat_results = flat_results.into_iter();
10991            let value = parallel_results_for_shape(&member.shape, &mut flat_results);
10992            match value.into_json_result() {
10993                Ok(value) => Some(value),
10994                Err(error) => return Poll::Ready(Err(error)),
10995            }
10996        } else {
10997            None
10998        };
10999        let winner = handles[member_position].clone();
11000        Poll::Ready(Ok(SelectionResult {
11001            key: winner.key.clone(),
11002            index: winner.index,
11003            kind: winner.kind.clone(),
11004            identity: winner.identity.clone(),
11005            value,
11006            failure: winner_failure,
11007            winner,
11008            handles,
11009        }))
11010    }
11011}
11012
11013fn selection_operation_identity(
11014    state: &WorkflowState,
11015    kind: &str,
11016    base_sequence: u64,
11017    size: usize,
11018) -> String {
11019    if kind == "group" {
11020        return format!("group:{base_sequence}:{size}");
11021    }
11022    let fields: &[&str] = match kind {
11023        "activity" => &["activity_execution_id"],
11024        "child" => &["child_workflow_run_id"],
11025        "timer" => &["timer_id"],
11026        "signal" => &["signal_wait_id"],
11027        "condition" => &["condition_wait_id"],
11028        _ => &[],
11029    };
11030    for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11031        for event in state
11032            .history_events
11033            .iter()
11034            .filter(|event| durable_event_sequence(event) == Some(sequence))
11035        {
11036            for field in fields {
11037                if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11038                    if !identity.is_empty() {
11039                        return identity.to_string();
11040                    }
11041                }
11042            }
11043        }
11044    }
11045    String::new()
11046}
11047
11048fn validated_selection_resolution_sequence(
11049    state: &WorkflowState,
11050    marker: &SelectionMarker,
11051    member: &SelectionMemberPlan,
11052    expected_identity: &str,
11053) -> Result<u64> {
11054    if expected_identity.is_empty() {
11055        return Err(invalid_recorded_history(
11056            "selection_operation_identity_missing",
11057            member.base_sequence,
11058            &format!(
11059                "durable {} resource identity from scheduled/open history",
11060                member.kind
11061            ),
11062            "missing operation identity",
11063            "selection member history is missing its canonical durable identity",
11064        ));
11065    }
11066    if marker.operation_identity != expected_identity {
11067        return Err(invalid_recorded_history(
11068            "selection_operation_identity_mismatch",
11069            member.base_sequence,
11070            expected_identity,
11071            &marker.operation_identity,
11072            "selection winner identity does not match durable scheduled/open history",
11073        ));
11074    }
11075
11076    let failure_types = [
11077        "ActivityFailed",
11078        "ActivityCancelled",
11079        "ActivityTimedOut",
11080        "ChildRunFailed",
11081        "ChildRunCancelled",
11082        "ChildRunTerminated",
11083    ];
11084    let success_types = [
11085        "ActivityCompleted",
11086        "ChildRunCompleted",
11087        "TimerFired",
11088        "SignalApplied",
11089        "ConditionWaitSatisfied",
11090        "ConditionWaitTimedOut",
11091    ];
11092    let terminal_types: &[&str] = if marker.outcome == "failed" {
11093        &failure_types
11094    } else {
11095        &success_types
11096    };
11097    let mut candidates = Vec::new();
11098    for event in state.history_events.iter() {
11099        let Some(sequence) = durable_event_sequence(event) else {
11100            continue;
11101        };
11102        if sequence < member.base_sequence
11103            || sequence >= member.base_sequence.saturating_add(member.size as u64)
11104            || !terminal_types.contains(&event.event_type.as_str())
11105        {
11106            continue;
11107        }
11108        let event_id = event
11109            .raw
11110            .get("id")
11111            .or_else(|| event.raw.get("event_id"))
11112            .and_then(Value::as_str)
11113            .filter(|value| !value.is_empty())
11114            .ok_or_else(|| {
11115                invalid_recorded_history(
11116                    "selection_resolution_event_id_missing",
11117                    member.base_sequence,
11118                    "terminal selection history with a durable event id",
11119                    &event.payload.to_string(),
11120                    "selection terminal history cannot be bound to its winner marker",
11121                )
11122            })?;
11123        candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11124    }
11125    let resolution = if marker.outcome == "failed" {
11126        candidates.first()
11127    } else {
11128        candidates.last()
11129    };
11130    let Some((event_id, event_type, sequence)) = resolution else {
11131        return Err(invalid_recorded_history(
11132            "selection_resolution_event_missing",
11133            member.base_sequence,
11134            "terminal history for the selected member",
11135            &format!("{:?}", marker.member_key),
11136            "selection winner marker has no matching durable terminal event",
11137        ));
11138    };
11139    if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11140        return Err(invalid_recorded_history(
11141            "selection_resolution_event_mismatch",
11142            member.base_sequence,
11143            &format!("{event_type}:{event_id}"),
11144            &format!(
11145                "{}:{}",
11146                marker.resolution_event_type, marker.resolution_event_id
11147            ),
11148            "selection winner marker does not reference the event that made its member terminal",
11149        ));
11150    }
11151    Ok(*sequence)
11152}
11153
11154fn recorded_selection_member_outcome(
11155    state: &WorkflowState,
11156    handle: &DurableOperationHandle,
11157) -> Result<Option<ParallelResult>> {
11158    for event in state.history_events.iter() {
11159        let Some(sequence) = durable_event_sequence(event) else {
11160            continue;
11161        };
11162        if sequence < handle.base_sequence
11163            || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11164            || !matches!(
11165                event.event_type.as_str(),
11166                "ActivityFailed"
11167                    | "ActivityCancelled"
11168                    | "ActivityTimedOut"
11169                    | "ChildRunFailed"
11170                    | "ChildRunCancelled"
11171                    | "ChildRunTerminated"
11172            )
11173        {
11174            continue;
11175        }
11176        let Some(command) = state
11177            .recorded_commands
11178            .iter()
11179            .find(|command| command.sequence() == sequence)
11180        else {
11181            continue;
11182        };
11183        match command {
11184            RecordedCommand::Activity {
11185                outcome: Some(Err(failure)),
11186                ..
11187            } => return Err(Error::ActivityFailed(failure.clone())),
11188            RecordedCommand::ChildWorkflow {
11189                outcome: Some(Err(failure)),
11190                ..
11191            } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11192            _ => {}
11193        }
11194    }
11195
11196    let mut results = Vec::with_capacity(handle.size);
11197    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11198        let Some(command) = state
11199            .recorded_commands
11200            .iter()
11201            .find(|command| command.sequence() == sequence)
11202        else {
11203            return Ok(None);
11204        };
11205        let result = match command {
11206            RecordedCommand::Activity { outcome, .. } => match outcome {
11207                Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11208                Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11209                None => return Ok(None),
11210            },
11211            RecordedCommand::Timer { fired, .. } => {
11212                if !fired {
11213                    return Ok(None);
11214                }
11215                ParallelAvroResult::Timer
11216            }
11217            RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11218                Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11219                Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11220                None => return Ok(None),
11221            },
11222            RecordedCommand::SignalWait { value, .. } => match value {
11223                Some(value) => ParallelAvroResult::Signal(value.clone()),
11224                None => return Ok(None),
11225            },
11226            RecordedCommand::ConditionWait { result, .. } => match result {
11227                Some(result) => ParallelAvroResult::Condition(*result),
11228                None => return Ok(None),
11229            },
11230            other => {
11231                return Err(command_mismatch(
11232                    other,
11233                    format!("selected {} member", handle.kind),
11234                ))
11235            }
11236        };
11237        results.push(result);
11238    }
11239    let mut results = results.into_iter();
11240    parallel_results_for_shape(&handle.shape, &mut results)
11241        .into_json_result()
11242        .map(Some)
11243}
11244
11245fn recorded_selection_member_is_terminal(
11246    state: &WorkflowState,
11247    handle: &DurableOperationHandle,
11248) -> bool {
11249    let mut completed = 0usize;
11250    let mut all_completed = true;
11251    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11252        let Some(command) = state
11253            .recorded_commands
11254            .iter()
11255            .find(|command| command.sequence() == sequence)
11256        else {
11257            all_completed = false;
11258            continue;
11259        };
11260        let terminal = match command {
11261            RecordedCommand::Activity {
11262                outcome: Some(Err(_)),
11263                ..
11264            }
11265            | RecordedCommand::ChildWorkflow {
11266                outcome: Some(Err(_)),
11267                ..
11268            } => return true,
11269            RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11270            RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11271            RecordedCommand::Timer { fired, .. } => *fired,
11272            RecordedCommand::SignalWait { value, .. } => value.is_some(),
11273            RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11274            RecordedCommand::SearchAttributes { .. }
11275            | RecordedCommand::SideEffect { .. }
11276            | RecordedCommand::VersionMarker { .. }
11277            | RecordedCommand::Memo { .. } => false,
11278        };
11279        if !terminal {
11280            all_completed = false;
11281            continue;
11282        }
11283        completed += 1;
11284    }
11285    all_completed && completed == handle.size
11286}
11287
11288fn selection_cancellation_for_handle(
11289    state: &WorkflowState,
11290    handle: &DurableOperationHandle,
11291) -> Result<bool> {
11292    let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11293        recorded.selection_group_id == handle.selection_group_id
11294            && recorded.member_base_sequence == handle.base_sequence
11295    }) else {
11296        return Ok(false);
11297    };
11298    validate_selection_cancellation_marker(marker, handle)?;
11299    Ok(true)
11300}
11301
11302fn validate_selection_cancellations_for_handles(
11303    state: &WorkflowState,
11304    handles: &[DurableOperationHandle],
11305) -> Result<()> {
11306    let Some(group_id) = handles
11307        .first()
11308        .map(|handle| handle.selection_group_id.as_str())
11309    else {
11310        return Ok(());
11311    };
11312    for marker in state
11313        .cancelled_selection_members
11314        .iter()
11315        .filter(|marker| marker.selection_group_id == group_id)
11316    {
11317        let Some(handle) = handles
11318            .iter()
11319            .find(|handle| handle.base_sequence == marker.member_base_sequence)
11320        else {
11321            return Err(invalid_recorded_history(
11322                "selection_cancellation_member_mismatch",
11323                marker.member_base_sequence,
11324                "SelectionOperationCancelled matching an authored selection handle",
11325                &format!("{marker:?}"),
11326                "selection cancellation member base does not name an authored member",
11327            ));
11328        };
11329        validate_selection_cancellation_marker(marker, handle)?;
11330    }
11331    Ok(())
11332}
11333
11334fn validate_selection_cancellation_marker(
11335    marker: &SelectionCancellation,
11336    handle: &DurableOperationHandle,
11337) -> Result<()> {
11338    if marker.selection_group_id != handle.selection_group_id
11339        || marker.member_key != handle.key
11340        || marker.member_index != handle.index
11341        || marker.member_base_sequence != handle.base_sequence
11342        || marker.member_size != handle.size
11343        || marker.operation_kind != handle.kind
11344        || marker.operation_identity != handle.identity
11345    {
11346        return Err(invalid_recorded_history(
11347            "selection_cancellation_member_mismatch",
11348            handle.base_sequence,
11349            "SelectionOperationCancelled matching the authored selection handle",
11350            &format!("{marker:?}"),
11351            "selection cancellation history targets different authored member metadata",
11352        ));
11353    }
11354    Ok(())
11355}
11356
11357/// Future returned by [`DurableOperationHandle::await_result`].
11358pub struct DurableOperationAwaitCall {
11359    handle: DurableOperationHandle,
11360}
11361
11362impl Future for DurableOperationAwaitCall {
11363    type Output = Result<ParallelResult>;
11364
11365    fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11366        let state = match self.handle.ctx.state.lock() {
11367            Ok(state) => state,
11368            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11369        };
11370        match selection_cancellation_for_handle(&state, &self.handle) {
11371            Err(error) => return Poll::Ready(Err(error)),
11372            Ok(false) => {}
11373            Ok(true) => {
11374                return Poll::Ready(Err(Error::DurableOperationCancelled(
11375                    DurableOperationCancelled {
11376                        selection_group_id: self.handle.selection_group_id.clone(),
11377                        member_key: self.handle.key.clone(),
11378                        member_index: self.handle.index,
11379                        operation_kind: self.handle.kind.clone(),
11380                        operation_identity: self.handle.identity.clone(),
11381                    },
11382                )));
11383            }
11384        }
11385        match recorded_selection_member_outcome(&state, &self.handle) {
11386            Ok(Some(result)) => Poll::Ready(Ok(result)),
11387            Ok(None) => Poll::Pending,
11388            Err(error) => Poll::Ready(Err(error)),
11389        }
11390    }
11391}
11392
11393/// Future returned by [`DurableOperationHandle::cancel`].
11394pub struct CancelDurableOperationCall {
11395    handle: DurableOperationHandle,
11396    emitted: bool,
11397}
11398
11399impl Future for CancelDurableOperationCall {
11400    type Output = Result<()>;
11401
11402    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11403        let ctx = self.handle.ctx.clone();
11404        let mut state = match ctx.state.lock() {
11405            Ok(state) => state,
11406            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11407        };
11408        match selection_cancellation_for_handle(&state, &self.handle) {
11409            Err(error) => return Poll::Ready(Err(error)),
11410            Ok(true) => return Poll::Ready(Ok(())),
11411            Ok(false) => {}
11412        }
11413        if recorded_selection_member_is_terminal(&state, &self.handle) {
11414            return Poll::Ready(Ok(()));
11415        }
11416        if !self.emitted {
11417            state.commands.push(json!({
11418                "type": "cancel_selection_operation",
11419                "selection_group_id": self.handle.selection_group_id,
11420                "member_key": self.handle.key,
11421                "member_index": self.handle.index,
11422                "member_base_sequence": self.handle.base_sequence,
11423                "member_size": self.handle.size,
11424                "operation_kind": self.handle.kind,
11425                "operation_identity": self.handle.identity,
11426            }));
11427            self.emitted = true;
11428        }
11429        // The cancellation command is only a request. Do not expose workflow
11430        // state after cancel until committed SelectionOperationCancelled
11431        // history proves that the request won the race with member completion.
11432        Poll::Pending
11433    }
11434}
11435
11436trait PollNestedResultExt<T> {
11437    fn flatten_result(self) -> Poll<Result<T>>;
11438}
11439
11440impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11441    fn flatten_result(self) -> Poll<Result<T>> {
11442        match self {
11443            Poll::Ready(Ok(result)) => Poll::Ready(result),
11444            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11445            Poll::Pending => Poll::Pending,
11446        }
11447    }
11448}
11449
11450struct SagaCompensation {
11451    activity_type: String,
11452    options: ActivityOptions,
11453    arguments: AvroValue,
11454    registration_order: usize,
11455}
11456
11457/// Workflow-local deterministic saga compensation helper.
11458///
11459/// Register each compensation only after its forward step succeeds. Passing
11460/// the forward `Result` to [`Saga::finish`] runs compensations sequentially in
11461/// reverse registration order after any failure, including cooperative
11462/// cancellation. Each compensation is an ordinary durable activity, so replay,
11463/// duplicate delivery, and worker restart use existing history semantics.
11464pub struct Saga {
11465    ctx: WorkflowContext,
11466    compensations: Vec<SagaCompensation>,
11467}
11468
11469impl Saga {
11470    fn new(ctx: WorkflowContext) -> Self {
11471        Self {
11472            ctx,
11473            compensations: Vec::new(),
11474        }
11475    }
11476
11477    pub fn add_compensation<T: Serialize>(
11478        &mut self,
11479        activity_type: impl Into<String>,
11480        args: T,
11481    ) -> Result<&mut Self> {
11482        self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11483    }
11484
11485    pub fn add_compensation_with_options<T: Serialize>(
11486        &mut self,
11487        activity_type: impl Into<String>,
11488        options: ActivityOptions,
11489        args: T,
11490    ) -> Result<&mut Self> {
11491        let activity_type = activity_type.into();
11492        if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11493            return Err(Error::Codec(
11494                "saga compensation activity type must be non-empty without surrounding whitespace"
11495                    .to_string(),
11496            ));
11497        }
11498        options.validate().map_err(Error::InvalidActivityOptions)?;
11499        let arguments = AvroValue::from_serialize(&args)?;
11500        let registration_order = self.compensations.len() + 1;
11501        self.compensations.push(SagaCompensation {
11502            activity_type,
11503            options,
11504            arguments,
11505            registration_order,
11506        });
11507        Ok(self)
11508    }
11509
11510    /// Compensate `initiating_failure` and return the failure that must remain.
11511    pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11512        while let Some(compensation) = self.compensations.pop() {
11513            if let Err(compensation_failure) = self
11514                .ctx
11515                .activity_with_options(
11516                    compensation.activity_type.clone(),
11517                    compensation.options,
11518                    compensation.arguments,
11519                )
11520                .await
11521            {
11522                if workflow_task_integrity_error(&compensation_failure) {
11523                    return compensation_failure;
11524                }
11525                return Error::SagaCompensationFailed(SagaCompensationFailure {
11526                    initiating_failure: Box::new(initiating_failure),
11527                    compensation_failure: Box::new(compensation_failure),
11528                    compensation_activity_type: compensation.activity_type,
11529                    compensation_registration_order: compensation.registration_order,
11530                });
11531            }
11532        }
11533        initiating_failure
11534    }
11535
11536    /// Return a successful forward value or compensate and preserve its failure.
11537    pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11538        match outcome {
11539            Ok(value) => Ok(value),
11540            Err(error) => Err(self.compensate(error).await),
11541        }
11542    }
11543}
11544
11545pub struct ActivityCall {
11546    ctx: WorkflowContext,
11547    activity_type: String,
11548    options: ActivityOptions,
11549    args: Option<Result<AvroValue>>,
11550    scheduled: bool,
11551    parallel_group_path: Vec<ParallelGroupMetadata>,
11552}
11553
11554impl ActivityCall {
11555    fn poll_avro_value(
11556        mut self: Pin<&mut Self>,
11557        _cx: &mut TaskContext<'_>,
11558    ) -> Poll<Result<AvroValue>> {
11559        let ctx = self.ctx.clone();
11560        let mut state = match ctx.state.lock() {
11561            Ok(state) => state,
11562            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11563        };
11564
11565        if self.scheduled {
11566            return Poll::Pending;
11567        }
11568
11569        let options = match self.options.validate() {
11570            Ok(options) => options,
11571            Err(error) => {
11572                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11573            }
11574        };
11575        let task_queue = options
11576            .task_queue
11577            .clone()
11578            .unwrap_or_else(|| state.task_queue.clone());
11579        let current_recorded_options = RecordedActivityOptions {
11580            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11581            // Rust schedules ordinary durable activities. The server records a
11582            // non-null mode only for a specialized execution primitive.
11583            execution_mode: RecordedSnapshotValue::Known(None),
11584            retry_policy: current_activity_retry_snapshot(&options),
11585        };
11586
11587        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11588            let sequence = recorded.sequence();
11589            match recorded {
11590                RecordedCommand::Activity {
11591                    activity_type,
11592                    options: recorded_options,
11593                    outcome,
11594                    parallel_group_path,
11595                    ..
11596                } => {
11597                    if let Err(error) = ensure_parallel_path_matches(
11598                        sequence,
11599                        parallel_group_path.as_deref(),
11600                        &self.parallel_group_path,
11601                    ) {
11602                        return Poll::Ready(Err(error));
11603                    }
11604                    if let Some(recorded_type) = activity_type {
11605                        if recorded_type != self.activity_type {
11606                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11607                                ReplayFailure::new(
11608                                    "recorded_command_detail_mismatch",
11609                                    Some(sequence),
11610                                    Some(format!("activity:{recorded_type}")),
11611                                    Some(format!("activity:{}", self.activity_type)),
11612                                    "recorded activity type differs from the current workflow command",
11613                                ),
11614                            )));
11615                        }
11616                    }
11617                    if let Some(recorded_options) = recorded_options {
11618                        if !recorded_options
11619                            .task_queue
11620                            .matches_current(&current_recorded_options.task_queue)
11621                        {
11622                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11623                                ReplayFailure::new(
11624                                    "activity_task_queue_mismatch",
11625                                    Some(sequence),
11626                                    Some(activity_options_description(&recorded_options)),
11627                                    Some(activity_options_description(&current_recorded_options)),
11628                                    "recorded activity task queue differs from the current workflow command",
11629                                ),
11630                            )));
11631                        }
11632                        if !recorded_options
11633                            .execution_mode
11634                            .matches_current(&current_recorded_options.execution_mode)
11635                        {
11636                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11637                                ReplayFailure::new(
11638                                    "activity_execution_mode_mismatch",
11639                                    Some(sequence),
11640                                    Some(activity_options_description(&recorded_options)),
11641                                    Some(activity_options_description(&current_recorded_options)),
11642                                    "recorded activity execution mode differs from the current workflow command",
11643                                ),
11644                            )));
11645                        }
11646                        if !recorded_options
11647                            .retry_policy
11648                            .matches_current(&current_recorded_options.retry_policy)
11649                        {
11650                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11651                                ReplayFailure::new(
11652                                    "activity_retry_policy_mismatch",
11653                                    Some(sequence),
11654                                    Some(activity_options_description(&recorded_options)),
11655                                    Some(activity_options_description(&current_recorded_options)),
11656                                    "recorded activity retry policy differs from the current workflow command",
11657                                ),
11658                            )));
11659                        }
11660                    }
11661                    state.command_cursor += 1;
11662                    if let Some(outcome) = outcome {
11663                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11664                    }
11665                    state.matched_recorded_pending = true;
11666                    self.scheduled = true;
11667                    return Poll::Pending;
11668                }
11669                other => {
11670                    return Poll::Ready(Err(command_mismatch(
11671                        &other,
11672                        format!("activity:{}", self.activity_type),
11673                    )));
11674                }
11675            }
11676        }
11677
11678        if !self.scheduled {
11679            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11680                Ok(args) => args,
11681                Err(error) => return Poll::Ready(Err(error)),
11682            };
11683            let arguments = normalize_avro_arguments(args);
11684            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11685                Ok(envelope) => envelope,
11686                Err(error) => return Poll::Ready(Err(error)),
11687            };
11688
11689            let mut command = serde_json::Map::from_iter([
11690                ("type".to_string(), json!("schedule_activity")),
11691                (
11692                    "activity_type".to_string(),
11693                    json!(self.activity_type.clone()),
11694                ),
11695                ("queue".to_string(), json!(task_queue)),
11696                ("arguments".to_string(), envelope),
11697            ]);
11698            for (field, value) in [
11699                ("start_to_close_timeout", options.start_to_close_timeout),
11700                (
11701                    "schedule_to_start_timeout",
11702                    options.schedule_to_start_timeout,
11703                ),
11704                (
11705                    "schedule_to_close_timeout",
11706                    options.schedule_to_close_timeout,
11707                ),
11708                ("heartbeat_timeout", options.heartbeat_timeout),
11709            ] {
11710                if let Some(value) = value {
11711                    command.insert(field.to_string(), json!(value));
11712                }
11713            }
11714            if let Some(retry_policy) = options.retry_policy {
11715                command.insert("retry_policy".to_string(), retry_policy);
11716            }
11717            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11718            state.commands.push(Value::Object(command));
11719            self.scheduled = true;
11720        }
11721
11722        Poll::Pending
11723    }
11724}
11725
11726impl Future for ActivityCall {
11727    type Output = Result<Value>;
11728
11729    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11730        match self.poll_avro_value(cx) {
11731            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11732            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11733            Poll::Pending => Poll::Pending,
11734        }
11735    }
11736}
11737
11738/// Future returned by [`WorkflowContext::sleep`].
11739pub struct TimerCall {
11740    ctx: WorkflowContext,
11741    delay_seconds: Option<u64>,
11742    scheduled: bool,
11743    matched_pending: bool,
11744    parallel_group_path: Vec<ParallelGroupMetadata>,
11745}
11746
11747impl Future for TimerCall {
11748    type Output = Result<()>;
11749
11750    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11751        if self.matched_pending {
11752            return Poll::Pending;
11753        }
11754
11755        let ctx = self.ctx.clone();
11756        let Some(requested_delay) = self.delay_seconds else {
11757            return Poll::Ready(Err(Error::TimerDurationOverflow));
11758        };
11759        let mut state = match ctx.state.lock() {
11760            Ok(state) => state,
11761            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11762        };
11763
11764        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11765            match recorded {
11766                RecordedCommand::Timer {
11767                    sequence,
11768                    delay_seconds,
11769                    fired,
11770                    parallel_group_path,
11771                    ..
11772                } => {
11773                    if let Err(error) = ensure_parallel_path_matches(
11774                        sequence,
11775                        parallel_group_path.as_deref(),
11776                        &self.parallel_group_path,
11777                    ) {
11778                        return Poll::Ready(Err(error));
11779                    }
11780                    if delay_seconds != requested_delay {
11781                        return Poll::Ready(Err(Error::NonDeterministicReplay(
11782                            ReplayFailure::new(
11783                                "timer_delay_mismatch",
11784                                Some(sequence),
11785                                Some(format!("timer:{delay_seconds}s")),
11786                                Some(format!("timer:{requested_delay}s")),
11787                                "recorded timer delay differs from the current workflow command",
11788                            ),
11789                        )));
11790                    }
11791                    state.command_cursor += 1;
11792                    if fired {
11793                        return Poll::Ready(Ok(()));
11794                    }
11795                    state.matched_recorded_pending = true;
11796                    self.scheduled = true;
11797                    self.matched_pending = true;
11798                    return Poll::Pending;
11799                }
11800                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11801            }
11802        }
11803
11804        if !self.scheduled {
11805            let mut command = serde_json::Map::from_iter([
11806                ("type".to_string(), json!("start_timer")),
11807                ("delay_seconds".to_string(), json!(requested_delay)),
11808            ]);
11809            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11810            state.commands.push(Value::Object(command));
11811            self.scheduled = true;
11812        }
11813
11814        Poll::Pending
11815    }
11816}
11817
11818/// Future returned by [`WorkflowContext::wait_condition`].
11819pub struct ConditionWaitCall {
11820    ctx: WorkflowContext,
11821    options: ConditionWaitOptions,
11822    predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11823    occurrence_id: Option<String>,
11824    opened_wait: bool,
11825    parallel_group_path: Vec<ParallelGroupMetadata>,
11826}
11827
11828impl Future for ConditionWaitCall {
11829    type Output = Result<ConditionWaitResult>;
11830
11831    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11832        if self.opened_wait {
11833            return Poll::Pending;
11834        }
11835
11836        let options = match self.options.validate() {
11837            Ok(options) => options,
11838            Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11839        };
11840        let ctx = self.ctx.clone();
11841        let occurrence_id = match self.occurrence_id.as_ref() {
11842            Some(occurrence_id) => occurrence_id.clone(),
11843            None => {
11844                let mut state = match ctx.state.lock() {
11845                    Ok(state) => state,
11846                    Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11847                };
11848                let ordinal = state.condition_wait_occurrence_counter;
11849                state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11850                    Some(next) => next,
11851                    None => {
11852                        return Poll::Ready(Err(Error::WorkerLoop(
11853                            "condition wait occurrence counter overflowed".to_string(),
11854                        )))
11855                    }
11856                };
11857                let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11858                drop(state);
11859                self.occurrence_id = Some(occurrence_id.clone());
11860                occurrence_id
11861            }
11862        };
11863
11864        let recorded_result = {
11865            let mut state = match ctx.state.lock() {
11866                Ok(state) => state,
11867                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11868            };
11869            let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11870                drop(state);
11871                return self.poll_new_condition(options);
11872            };
11873            if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11874                return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11875            }
11876
11877            let mut cursor = state.command_cursor;
11878            let mut result = None;
11879            loop {
11880                let Some(RecordedCommand::ConditionWait {
11881                    sequence,
11882                    occurrence_id: recorded_occurrence_id,
11883                    condition_key,
11884                    predicate_identity,
11885                    timeout_seconds,
11886                    result: recorded_result,
11887                    parallel_group_path,
11888                    ..
11889                }) = state.recorded_commands.get(cursor)
11890                else {
11891                    break;
11892                };
11893
11894                if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
11895                    break;
11896                }
11897                if let Err(error) = ensure_parallel_path_matches(
11898                    *sequence,
11899                    parallel_group_path.as_deref(),
11900                    &self.parallel_group_path,
11901                ) {
11902                    return Poll::Ready(Err(error));
11903                }
11904                if let Err(error) = validate_recorded_condition_wait(
11905                    *sequence,
11906                    recorded_occurrence_id,
11907                    condition_key.as_deref(),
11908                    predicate_identity,
11909                    *timeout_seconds,
11910                    &occurrence_id,
11911                    &options,
11912                ) {
11913                    return Poll::Ready(Err(error));
11914                }
11915                if result == Some(ConditionWaitResult::TimedOut) {
11916                    return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
11917                        "condition_wait_reopened_after_timeout",
11918                        Some(*sequence),
11919                        Some("timed-out condition is terminal".to_string()),
11920                        Some("another physical wait-open".to_string()),
11921                        "condition history reopened one logical wait after its durable timeout",
11922                    ))));
11923                }
11924                result = *recorded_result;
11925                cursor += 1;
11926            }
11927            state.command_cursor = cursor;
11928            result
11929        };
11930
11931        if let Some(result) = recorded_result {
11932            return Poll::Ready(Ok(result));
11933        }
11934
11935        self.poll_open_condition(options)
11936    }
11937}
11938
11939impl ConditionWaitCall {
11940    fn poll_new_condition(
11941        self: Pin<&mut Self>,
11942        options: ValidatedConditionWaitOptions,
11943    ) -> Poll<Result<ConditionWaitResult>> {
11944        self.poll_open_condition(options)
11945    }
11946
11947    fn poll_open_condition(
11948        mut self: Pin<&mut Self>,
11949        options: ValidatedConditionWaitOptions,
11950    ) -> Poll<Result<ConditionWaitResult>> {
11951        let selection_member = self
11952            .parallel_group_path
11953            .first()
11954            .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
11955        match (self.predicate)() {
11956            Ok(true) if !selection_member => {
11957                return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
11958            }
11959            Ok(_) => {}
11960            Err(error) => return Poll::Ready(Err(error)),
11961        }
11962        if options.timeout_seconds == Some(0) && !selection_member {
11963            return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
11964        }
11965
11966        let ctx = self.ctx.clone();
11967        let mut state = match ctx.state.lock() {
11968            Ok(state) => state,
11969            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11970        };
11971        let mut command = serde_json::Map::from_iter([
11972            ("type".to_string(), json!("open_condition_wait")),
11973            (
11974                "condition_wait_occurrence_id".to_string(),
11975                json!(self.occurrence_id.as_deref().unwrap_or_default()),
11976            ),
11977            ("condition_key".to_string(), json!(options.condition_key)),
11978            (
11979                "condition_definition_fingerprint".to_string(),
11980                json!(options.predicate_identity),
11981            ),
11982        ]);
11983        if let Some(timeout_seconds) = options.timeout_seconds {
11984            command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
11985        }
11986        apply_parallel_group_path(&mut command, &self.parallel_group_path);
11987        state.commands.push(Value::Object(command));
11988        drop(state);
11989        self.opened_wait = true;
11990        Poll::Pending
11991    }
11992}
11993
11994fn validate_recorded_condition_wait(
11995    sequence: u64,
11996    recorded_occurrence_id: &str,
11997    recorded_key: Option<&str>,
11998    recorded_predicate_identity: &str,
11999    recorded_timeout_seconds: Option<u64>,
12000    current_occurrence_id: &str,
12001    current: &ValidatedConditionWaitOptions,
12002) -> Result<()> {
12003    if recorded_occurrence_id != current_occurrence_id {
12004        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12005            "condition_wait_occurrence_mismatch",
12006            Some(sequence),
12007            Some(recorded_occurrence_id.to_string()),
12008            Some(current_occurrence_id.to_string()),
12009            "recorded condition occurrence differs from the current authored wait position",
12010        )));
12011    }
12012    if recorded_key != Some(current.condition_key.as_str()) {
12013        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12014            "condition_wait_key_mismatch",
12015            Some(sequence),
12016            recorded_key.map(str::to_string),
12017            Some(current.condition_key.clone()),
12018            "recorded condition identity differs from the current workflow wait",
12019        )));
12020    }
12021    if recorded_predicate_identity != current.predicate_identity {
12022        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12023            "condition_wait_predicate_mismatch",
12024            Some(sequence),
12025            Some(recorded_predicate_identity.to_string()),
12026            Some(current.predicate_identity.clone()),
12027            "recorded condition predicate behavior differs from current workflow code",
12028        )));
12029    }
12030    if recorded_timeout_seconds != current.timeout_seconds {
12031        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12032            "condition_wait_timeout_mismatch",
12033            Some(sequence),
12034            recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12035            current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12036            "recorded condition timeout differs from the current workflow wait",
12037        )));
12038    }
12039    Ok(())
12040}
12041
12042/// Future returned by [`WorkflowContext::start_child_workflow`].
12043pub struct ChildWorkflowCall {
12044    ctx: WorkflowContext,
12045    workflow_type: String,
12046    options: ChildWorkflowOptions,
12047    args: Option<Result<AvroValue>>,
12048    scheduled: bool,
12049    matched_pending: bool,
12050    parallel_group_path: Vec<ParallelGroupMetadata>,
12051}
12052
12053impl ChildWorkflowCall {
12054    fn poll_avro_value(
12055        mut self: Pin<&mut Self>,
12056        _cx: &mut TaskContext<'_>,
12057    ) -> Poll<Result<ChildWorkflowAvroResult>> {
12058        if self.matched_pending {
12059            return Poll::Pending;
12060        }
12061
12062        let ctx = self.ctx.clone();
12063        let mut state = match ctx.state.lock() {
12064            Ok(state) => state,
12065            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12066        };
12067
12068        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12069            let sequence = recorded.sequence();
12070            match recorded {
12071                RecordedCommand::ChildWorkflow {
12072                    workflow_type,
12073                    outcome,
12074                    parallel_group_path,
12075                    ..
12076                } => {
12077                    if let Err(error) = ensure_parallel_path_matches(
12078                        sequence,
12079                        parallel_group_path.as_deref(),
12080                        &self.parallel_group_path,
12081                    ) {
12082                        return Poll::Ready(Err(error));
12083                    }
12084                    if let Some(recorded_type) = workflow_type {
12085                        if recorded_type != self.workflow_type {
12086                            return Poll::Ready(Err(Error::NonDeterministicReplay(
12087                                ReplayFailure::new(
12088                                    "recorded_command_detail_mismatch",
12089                                    Some(sequence),
12090                                    Some(format!("child workflow:{recorded_type}")),
12091                                    Some(format!("child workflow:{}", self.workflow_type)),
12092                                    "recorded child workflow type differs from the current workflow command",
12093                                ),
12094                            )));
12095                        }
12096                    }
12097                    state.command_cursor += 1;
12098                    if let Some(outcome) = outcome {
12099                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12100                    }
12101                    state.matched_recorded_pending = true;
12102                    self.scheduled = true;
12103                    self.matched_pending = true;
12104                    return Poll::Pending;
12105                }
12106                other => {
12107                    return Poll::Ready(Err(command_mismatch(
12108                        &other,
12109                        format!("child workflow:{}", self.workflow_type),
12110                    )));
12111                }
12112            }
12113        }
12114
12115        if !self.scheduled {
12116            if self.options.task_queue.trim().is_empty() {
12117                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12118                    "task_queue must not be empty".to_string(),
12119                )));
12120            }
12121            for (name, value) in [
12122                (
12123                    "execution_timeout_seconds",
12124                    self.options.execution_timeout_seconds,
12125                ),
12126                ("run_timeout_seconds", self.options.run_timeout_seconds),
12127            ] {
12128                if value == Some(0) {
12129                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12130                        "{name} must be at least 1"
12131                    ))));
12132                }
12133            }
12134
12135            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12136                Ok(args) => args,
12137                Err(error) => return Poll::Ready(Err(error)),
12138            };
12139            let arguments = match encode_typed_envelope(
12140                &normalize_avro_arguments(args),
12141                &state.payload_codec,
12142            ) {
12143                Ok(arguments) => arguments,
12144                Err(error) => return Poll::Ready(Err(error)),
12145            };
12146            let mut command = json!({
12147                "type": "start_child_workflow",
12148                "workflow_type": self.workflow_type,
12149                "queue": self.options.task_queue,
12150                "parent_close_policy": self.options.parent_close_policy.as_str(),
12151                "arguments": arguments,
12152            });
12153            let object = command
12154                .as_object_mut()
12155                .expect("child workflow command is always an object");
12156            if let Some(policy) = &self.options.retry_policy {
12157                let mut retry_policy = serde_json::Map::new();
12158                if let Some(max_attempts) = policy.max_attempts {
12159                    if max_attempts == 0 {
12160                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12161                            "retry_policy.max_attempts must be at least 1".to_string(),
12162                        )));
12163                    }
12164                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12165                }
12166                if !policy.backoff_seconds.is_empty() {
12167                    retry_policy
12168                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12169                }
12170                if !policy.non_retryable_error_types.is_empty() {
12171                    retry_policy.insert(
12172                        "non_retryable_error_types".to_string(),
12173                        json!(policy.non_retryable_error_types),
12174                    );
12175                }
12176                if retry_policy.is_empty() {
12177                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12178                        "retry_policy must configure at least one field".to_string(),
12179                    )));
12180                }
12181                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12182            }
12183            if let Some(seconds) = self.options.execution_timeout_seconds {
12184                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12185            }
12186            if let Some(seconds) = self.options.run_timeout_seconds {
12187                object.insert("run_timeout_seconds".to_string(), json!(seconds));
12188            }
12189            apply_parallel_group_path(object, &self.parallel_group_path);
12190            state.commands.push(command);
12191            self.scheduled = true;
12192        }
12193
12194        Poll::Pending
12195    }
12196}
12197
12198impl Future for ChildWorkflowCall {
12199    type Output = Result<ChildWorkflowResult>;
12200
12201    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12202        match self.poll_avro_value(cx) {
12203            Poll::Ready(Ok(result)) => match result.result.into_json() {
12204                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12205                    parent: result.parent,
12206                    child: result.child,
12207                    child_workflow_type: result.child_workflow_type,
12208                    result: projected,
12209                })),
12210                Err(error) => Poll::Ready(Err(error)),
12211            },
12212            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12213            Poll::Pending => Poll::Pending,
12214        }
12215    }
12216}
12217
12218fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12219    Error::NonDeterministicReplay(ReplayFailure::new(
12220        "recorded_command_mismatch",
12221        Some(recorded.sequence()),
12222        Some(recorded.shape().to_string()),
12223        Some(actual.into()),
12224        "current workflow command does not match the recorded durable command sequence",
12225    ))
12226}
12227
12228pub struct SignalCall {
12229    ctx: WorkflowContext,
12230    signal_name: String,
12231    runtime_reserved_allowed: bool,
12232    opened_wait: bool,
12233    matched_pending: bool,
12234    parallel_group_path: Vec<ParallelGroupMetadata>,
12235}
12236
12237impl SignalCall {
12238    fn poll_avro_value(
12239        mut self: Pin<&mut Self>,
12240        _cx: &mut TaskContext<'_>,
12241    ) -> Poll<Result<Vec<AvroValue>>> {
12242        if self.matched_pending {
12243            return Poll::Pending;
12244        }
12245        if !self.runtime_reserved_allowed {
12246            if let Err(error) = validate_user_signal_name(&self.signal_name) {
12247                return Poll::Ready(Err(error));
12248            }
12249        }
12250
12251        let ctx = self.ctx.clone();
12252        let mut state = match ctx.state.lock() {
12253            Ok(state) => state,
12254            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12255        };
12256
12257        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12258            match recorded {
12259                RecordedCommand::SignalWait {
12260                    sequence,
12261                    signal_name,
12262                    value,
12263                    parallel_group_path,
12264                } => {
12265                    if let Err(error) = ensure_parallel_path_matches(
12266                        sequence,
12267                        parallel_group_path.as_deref(),
12268                        &self.parallel_group_path,
12269                    ) {
12270                        return Poll::Ready(Err(error));
12271                    }
12272                    if signal_name != self.signal_name {
12273                        return Poll::Ready(Err(Error::NonDeterministicReplay(
12274                            ReplayFailure::new(
12275                                "recorded_command_detail_mismatch",
12276                                Some(sequence),
12277                                Some(format!("signal wait:{signal_name}")),
12278                                Some(format!("signal wait:{}", self.signal_name)),
12279                                "recorded signal name differs from the current workflow command",
12280                            ),
12281                        )));
12282                    }
12283
12284                    state.command_cursor += 1;
12285                    if let Some(value) = value {
12286                        return Poll::Ready(Ok(value));
12287                    }
12288                    if state
12289                        .resume_signal
12290                        .as_ref()
12291                        .is_some_and(|signal| signal.signal_name == self.signal_name)
12292                    {
12293                        let signal = state
12294                            .resume_signal
12295                            .take()
12296                            .expect("matching resume signal is present");
12297                        return Poll::Ready(Ok(signal.arguments));
12298                    }
12299
12300                    state.matched_recorded_pending = true;
12301                    self.opened_wait = true;
12302                    self.matched_pending = true;
12303                    return Poll::Pending;
12304                }
12305                other => {
12306                    return Poll::Ready(Err(command_mismatch(
12307                        &other,
12308                        format!("signal wait:{}", self.signal_name),
12309                    )));
12310                }
12311            }
12312        }
12313
12314        if state
12315            .resume_signal
12316            .as_ref()
12317            .is_some_and(|signal| signal.signal_name == self.signal_name)
12318        {
12319            let signal = state
12320                .resume_signal
12321                .take()
12322                .expect("matching resume signal is present");
12323            return Poll::Ready(Ok(signal.arguments));
12324        }
12325
12326        if !self.opened_wait {
12327            let mut command = serde_json::Map::from_iter([
12328                ("type".to_string(), json!("open_signal_wait")),
12329                ("signal_name".to_string(), json!(self.signal_name)),
12330            ]);
12331            apply_parallel_group_path(&mut command, &self.parallel_group_path);
12332            state.commands.push(Value::Object(command));
12333            self.opened_wait = true;
12334        }
12335
12336        Poll::Pending
12337    }
12338}
12339
12340impl Future for SignalCall {
12341    type Output = Result<Vec<Value>>;
12342
12343    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12344        match self.poll_avro_value(cx) {
12345            Poll::Ready(Ok(values)) => Poll::Ready(
12346                values
12347                    .into_iter()
12348                    .map(AvroValue::into_json)
12349                    .collect::<Result<Vec<_>>>(),
12350            ),
12351            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12352            Poll::Pending => Poll::Pending,
12353        }
12354    }
12355}
12356
12357#[derive(Clone, Debug)]
12358pub struct ActivityContext {
12359    client: Client,
12360    pub task_id: String,
12361    pub activity_attempt_id: String,
12362    pub lease_owner: String,
12363    pub activity_type: String,
12364    pub attempt_number: u64,
12365    pub task_queue: String,
12366    pub worker_id: String,
12367}
12368
12369impl ActivityContext {
12370    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12371        self.client
12372            .heartbeat_activity_task(
12373                &self.task_id,
12374                &self.activity_attempt_id,
12375                &self.lease_owner,
12376                details,
12377            )
12378            .await
12379    }
12380}
12381
12382fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12383    validate_payload_codec(codec)?;
12384    match value {
12385        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12386            value, codec,
12387        )?)),
12388        None => Ok(AvroValue::Array(Vec::new())),
12389    }
12390}
12391
12392fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12393    let Some(signal_name) = task
12394        .signal_name
12395        .as_deref()
12396        .filter(|value| !value.is_empty())
12397    else {
12398        return Ok(None);
12399    };
12400    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12401    let AvroValue::Array(arguments) = decoded else {
12402        unreachable!("normalize_avro_arguments always returns an array");
12403    };
12404
12405    Ok(Some(ResumeSignal {
12406        signal_name: signal_name.to_string(),
12407        arguments,
12408    }))
12409}
12410
12411fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12412    validate_payload_codec(&task.payload_codec)?;
12413    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12414    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12415    for event in &task.history_events {
12416        validate_history_event_payloads(event, &task.payload_codec)?;
12417    }
12418    Ok(())
12419}
12420
12421fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12422    validate_payload_codec(&task.payload_codec)?;
12423    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12424}
12425
12426fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12427    validate_payload_codec(&task.payload_codec)?;
12428    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12429    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12430    for event in &task.history_events {
12431        validate_history_event_payloads(event, &task.payload_codec)?;
12432    }
12433
12434    let Some(export) = task.history_export.as_ref() else {
12435        return Ok(());
12436    };
12437    let export_codec = match export.get("payloads") {
12438        Some(payloads) => declared_payload_codec(payloads, "codec")?,
12439        None => None,
12440    }
12441    .unwrap_or(&task.payload_codec);
12442    validate_payload_codec(export_codec)?;
12443
12444    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12445        for event in events {
12446            let event_type = event
12447                .get("event_type")
12448                .or_else(|| event.get("type"))
12449                .and_then(Value::as_str)
12450                .unwrap_or_default();
12451            if let Some(payload) = event.get("payload") {
12452                validate_history_payloads(event_type, payload, export_codec)?;
12453            }
12454        }
12455    }
12456    for signal in export
12457        .get("signals")
12458        .and_then(Value::as_array)
12459        .into_iter()
12460        .flatten()
12461    {
12462        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12463        validate_payload_codec(codec)?;
12464        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12465    }
12466    for activity in export
12467        .get("activities")
12468        .and_then(Value::as_array)
12469        .into_iter()
12470        .flatten()
12471    {
12472        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12473        validate_payload_codec(codec)?;
12474        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12475        validate_optional_inbound_payload(activity.get("result"), codec)?;
12476    }
12477    Ok(())
12478}
12479
12480fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12481    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12482}
12483
12484fn validate_history_payloads(
12485    event_type: &str,
12486    payload: &Value,
12487    fallback_codec: &str,
12488) -> Result<()> {
12489    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12490    validate_payload_codec(codec)?;
12491    for field in history_payload_fields(event_type) {
12492        validate_optional_inbound_payload(payload.get(*field), codec)?;
12493    }
12494    Ok(())
12495}
12496
12497const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12498
12499fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12500    match event_type {
12501        "ActivityCompleted" => &["result"],
12502        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12503        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12504        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12505        "ChildRunCompleted" => &["result", "output"],
12506        "WorkflowCompleted" => &["output"],
12507        "ServiceCallStarted"
12508        | "ServiceCallCompleted"
12509        | "ServiceCallFailed"
12510        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12511        _ => &[],
12512    }
12513}
12514
12515fn signal_history_payload(payload: &Value) -> Option<&Value> {
12516    SIGNAL_HISTORY_PAYLOAD_FIELDS
12517        .iter()
12518        .find_map(|field| payload.get(*field))
12519}
12520
12521fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12522    match value.get(field) {
12523        None => Ok(None),
12524        Some(Value::String(codec)) => Ok(Some(codec)),
12525        Some(_) => Err(invalid_payload_envelope()),
12526    }
12527}
12528
12529fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12530    validate_payload_codec(codec)?;
12531    if let Some(value) = value.filter(|value| !value.is_null()) {
12532        decode_wire_avro_value(value, codec)?;
12533    }
12534    Ok(())
12535}
12536
12537fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12538    let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12539        invalid_recorded_history(
12540            "parallel_group_metadata_invalid",
12541            sequence,
12542            "non-empty parallel_group_id",
12543            &payload.to_string(),
12544            "parallel-group history is missing its stable identity",
12545        )
12546    })?;
12547    let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12548        invalid_recorded_history(
12549            "parallel_group_metadata_invalid",
12550            sequence,
12551            "activity, child, timer, signal, condition, or mixed group kind",
12552            &payload.to_string(),
12553            "parallel-group history is missing its group kind",
12554        )
12555    })?;
12556    if !matches!(
12557        kind.as_str(),
12558        "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12559    ) {
12560        return Err(invalid_recorded_history(
12561            "parallel_group_metadata_invalid",
12562            sequence,
12563            "activity, child, timer, signal, condition, or mixed group kind",
12564            &kind,
12565            "parallel-group history contains an unsupported group kind",
12566        ));
12567    }
12568    let base_sequence = payload
12569        .get("parallel_group_base_sequence")
12570        .and_then(value_as_u64)
12571        .filter(|value| *value > 0)
12572        .ok_or_else(|| {
12573            invalid_recorded_history(
12574                "parallel_group_metadata_invalid",
12575                sequence,
12576                "positive parallel_group_base_sequence",
12577                &payload.to_string(),
12578                "parallel-group history contains an invalid base sequence",
12579            )
12580        })?;
12581    let size = payload
12582        .get("parallel_group_size")
12583        .and_then(value_as_u64)
12584        .and_then(|value| usize::try_from(value).ok())
12585        .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12586        .ok_or_else(|| {
12587            invalid_recorded_history(
12588                "parallel_group_metadata_invalid",
12589                sequence,
12590                "bounded positive parallel_group_size",
12591                &payload.to_string(),
12592                "parallel-group history contains an invalid group size",
12593            )
12594        })?;
12595    let index = payload
12596        .get("parallel_group_index")
12597        .and_then(value_as_u64)
12598        .and_then(|value| usize::try_from(value).ok())
12599        .filter(|value| *value < size)
12600        .ok_or_else(|| {
12601            invalid_recorded_history(
12602                "parallel_group_metadata_invalid",
12603                sequence,
12604                "parallel_group_index within group bounds",
12605                &payload.to_string(),
12606                "parallel-group history contains an invalid member index",
12607            )
12608        })?;
12609    if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12610        return Err(invalid_recorded_history(
12611            "parallel_group_metadata_invalid",
12612            sequence,
12613            "base sequence plus member index equals workflow sequence",
12614            &payload.to_string(),
12615            "parallel-group path does not preserve durable workflow position",
12616        ));
12617    }
12618    let mode = payload
12619        .get("parallel_group_mode")
12620        .and_then(Value::as_str)
12621        .unwrap_or("all");
12622    if !matches!(mode, "all" | "select") {
12623        return Err(invalid_recorded_history(
12624            "parallel_group_metadata_invalid",
12625            sequence,
12626            "parallel group mode all or select",
12627            mode,
12628            "parallel-group history contains an unsupported group mode",
12629        ));
12630    }
12631    let expected_id = if mode == "select" {
12632        format!("select-calls:{base_sequence}:{size}")
12633    } else {
12634        format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12635    };
12636    if group_id != expected_id {
12637        return Err(invalid_recorded_history(
12638            "parallel_group_metadata_invalid",
12639            sequence,
12640            &expected_id,
12641            &group_id,
12642            "parallel-group history contains an incompatible stable group ID",
12643        ));
12644    }
12645    let selection_member_key = if mode == "select" {
12646        Some(selection_key_from_value(
12647            payload.get("selection_member_key"),
12648            sequence,
12649        )?)
12650    } else {
12651        None
12652    };
12653    let selection_member_index = if mode == "select" {
12654        Some(required_parallel_usize(
12655            payload,
12656            "selection_member_index",
12657            sequence,
12658        )?)
12659    } else {
12660        None
12661    };
12662    let selection_member_base_sequence = if mode == "select" {
12663        Some(
12664            payload
12665                .get("selection_member_base_sequence")
12666                .and_then(value_as_u64)
12667                .filter(|value| *value >= base_sequence)
12668                .ok_or_else(|| {
12669                    invalid_recorded_history(
12670                        "parallel_group_metadata_invalid",
12671                        sequence,
12672                        "selection member base within its group",
12673                        &payload.to_string(),
12674                        "selection history contains an invalid member base sequence",
12675                    )
12676                })?,
12677        )
12678    } else {
12679        None
12680    };
12681    let selection_member_size = if mode == "select" {
12682        let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12683        if member_size == 0 {
12684            return Err(invalid_recorded_history(
12685                "parallel_group_metadata_invalid",
12686                sequence,
12687                "positive selection member size",
12688                &payload.to_string(),
12689                "selection history contains an invalid member size",
12690            ));
12691        }
12692        Some(member_size)
12693    } else {
12694        None
12695    };
12696    let selection_member_kind = if mode == "select" {
12697        let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12698            invalid_recorded_history(
12699                "parallel_group_metadata_invalid",
12700                sequence,
12701                "selection member operation kind",
12702                &payload.to_string(),
12703                "selection history is missing its authored member kind",
12704            )
12705        })?;
12706        if !matches!(
12707            kind.as_str(),
12708            "activity" | "child" | "timer" | "signal" | "condition" | "group"
12709        ) {
12710            return Err(invalid_recorded_history(
12711                "parallel_group_metadata_invalid",
12712                sequence,
12713                "activity, child, timer, signal, condition, or group selection member kind",
12714                &kind,
12715                "selection history contains an unsupported member kind",
12716            ));
12717        }
12718        Some(kind)
12719    } else {
12720        None
12721    };
12722    if let (Some(member_base), Some(member_size)) =
12723        (selection_member_base_sequence, selection_member_size)
12724    {
12725        let member_end = member_base
12726            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12727            .ok_or_else(|| {
12728                invalid_recorded_history(
12729                    "parallel_group_metadata_invalid",
12730                    sequence,
12731                    "bounded selection member range",
12732                    &payload.to_string(),
12733                    "selection member range overflowed",
12734                )
12735            })?;
12736        let group_end = base_sequence
12737            .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12738            .unwrap_or(u64::MAX);
12739        if sequence < member_base || sequence >= member_end || member_end > group_end {
12740            return Err(invalid_recorded_history(
12741                "parallel_group_metadata_invalid",
12742                sequence,
12743                "workflow sequence within one bounded selection member",
12744                &payload.to_string(),
12745                "selection member range does not contain its durable leaf",
12746            ));
12747        }
12748    }
12749    Ok(ParallelGroupMetadata {
12750        parallel_group_id: group_id,
12751        parallel_group_kind: kind,
12752        parallel_group_base_sequence: base_sequence,
12753        parallel_group_size: size,
12754        parallel_group_index: index,
12755        parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12756        selection_member_key,
12757        selection_member_index,
12758        selection_member_base_sequence,
12759        selection_member_size,
12760        selection_member_kind,
12761    })
12762}
12763
12764fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12765    payload
12766        .get(field)
12767        .and_then(value_as_u64)
12768        .and_then(|value| usize::try_from(value).ok())
12769        .ok_or_else(|| {
12770            invalid_recorded_history(
12771                "parallel_group_metadata_invalid",
12772                sequence,
12773                &format!("non-negative integer {field}"),
12774                &payload.to_string(),
12775                "selection history contains invalid member metadata",
12776            )
12777        })
12778}
12779
12780fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12781    match value {
12782        Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12783        Some(value) => value_as_u64(value)
12784            .and_then(|value| usize::try_from(value).ok())
12785            .map(SelectionKey::Index)
12786            .ok_or_else(|| {
12787                invalid_recorded_history(
12788                    "selection_member_key_invalid",
12789                    sequence,
12790                    "non-empty string or non-negative integer member key",
12791                    &value.to_string(),
12792                    "selection history contains an invalid member key",
12793                )
12794            }),
12795        None => Err(invalid_recorded_history(
12796            "selection_member_key_missing",
12797            sequence,
12798            "selection_member_key",
12799            "<missing>",
12800            "selection history is missing its stable member key",
12801        )),
12802    }
12803}
12804
12805fn recorded_parallel_group_path(
12806    events: &[&HistoryEvent],
12807    sequence: u64,
12808) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12809    let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12810    for event in events {
12811        let payload = &event.payload;
12812        let has_metadata = payload.get("parallel_group_path").is_some()
12813            || payload.get("parallel_group_id").is_some()
12814            || payload.get("parallel_group_kind").is_some()
12815            || payload.get("parallel_group_base_sequence").is_some()
12816            || payload.get("parallel_group_size").is_some()
12817            || payload.get("parallel_group_index").is_some()
12818            || payload.get("parallel_group_mode").is_some()
12819            || payload.get("selection_member_key").is_some();
12820        if !has_metadata {
12821            continue;
12822        }
12823
12824        let top_level = recorded_parallel_group_entry(payload, sequence)?;
12825        let path = match payload.get("parallel_group_path") {
12826            None => vec![top_level.clone()],
12827            Some(Value::Array(entries)) if !entries.is_empty() => entries
12828                .iter()
12829                .map(|entry| recorded_parallel_group_entry(entry, sequence))
12830                .collect::<Result<Vec<_>>>()?,
12831            Some(value) => {
12832                return Err(invalid_recorded_history(
12833                    "parallel_group_metadata_invalid",
12834                    sequence,
12835                    "non-empty parallel_group_path list",
12836                    &value.to_string(),
12837                    "parallel-group history contains an invalid group path",
12838                ));
12839            }
12840        };
12841        if path.last() != Some(&top_level) {
12842            return Err(invalid_recorded_history(
12843                "parallel_group_metadata_invalid",
12844                sequence,
12845                &serde_json::to_string(&path.last()).unwrap_or_default(),
12846                &serde_json::to_string(&top_level).unwrap_or_default(),
12847                "parallel-group top-level fields do not match the innermost path entry",
12848            ));
12849        }
12850        if recorded.as_ref().is_some_and(|existing| existing != &path) {
12851            return Err(invalid_recorded_history(
12852                "parallel_group_history_conflict",
12853                sequence,
12854                &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12855                &serde_json::to_string(&path).unwrap_or_default(),
12856                "parallel-group metadata changed between scheduling and resolution history",
12857            ));
12858        }
12859        recorded = Some(path);
12860    }
12861    Ok(recorded)
12862}
12863
12864fn recorded_commands(
12865    events: &[HistoryEvent],
12866    fallback_codec: &str,
12867    parent: WorkflowIdentity,
12868) -> Result<Vec<RecordedCommand>> {
12869    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12870    let mut last_new_sequence = None;
12871
12872    for event in events {
12873        let is_activity = matches!(
12874            event.event_type.as_str(),
12875            "ActivityScheduled"
12876                | "ActivityStarted"
12877                | "ActivityHeartbeatRecorded"
12878                | "ActivityRetryScheduled"
12879                | "ActivityCompleted"
12880                | "ActivityFailed"
12881                | "ActivityCancelled"
12882                | "ActivityTimedOut"
12883        );
12884        let is_workflow_timer = matches!(
12885            event.event_type.as_str(),
12886            "TimerScheduled" | "TimerCancelled" | "TimerFired"
12887        ) && !is_internal_timer_event(event);
12888        let is_child_workflow = matches!(
12889            event.event_type.as_str(),
12890            "ChildWorkflowScheduled"
12891                | "ChildRunCompleted"
12892                | "ChildRunFailed"
12893                | "ChildRunCancelled"
12894                | "ChildRunTerminated"
12895        );
12896        let is_signal_wait = is_recorded_signal_wait_event(event);
12897        let is_condition_wait = is_recorded_condition_wait_event(event);
12898        let is_search_attributes = event.event_type == "SearchAttributesUpserted";
12899        let is_side_effect = event.event_type == "SideEffectRecorded";
12900        let is_version_marker = event.event_type == "VersionMarkerRecorded";
12901        let is_memo = event.event_type == "MemoUpserted";
12902        if !is_activity
12903            && !is_workflow_timer
12904            && !is_child_workflow
12905            && !is_signal_wait
12906            && !is_condition_wait
12907            && !is_search_attributes
12908            && !is_side_effect
12909            && !is_version_marker
12910            && !is_memo
12911        {
12912            continue;
12913        }
12914
12915        let sequence = durable_event_sequence(event).ok_or_else(|| {
12916            Error::NonDeterministicReplay(ReplayFailure::new(
12917                "durable_command_sequence_missing",
12918                None,
12919                Some("positive workflow sequence".to_string()),
12920                Some(event.event_type.clone()),
12921                "durable command history event has no workflow sequence",
12922            ))
12923        })?;
12924        if sequence == 0 {
12925            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12926                "durable_command_sequence_invalid",
12927                Some(sequence),
12928                Some("positive workflow sequence".to_string()),
12929                Some(sequence.to_string()),
12930                "durable command history uses an invalid workflow sequence",
12931            )));
12932        }
12933        if !events_by_sequence.contains_key(&sequence) {
12934            if let Some(previous) = last_new_sequence {
12935                if sequence < previous {
12936                    return Err(invalid_recorded_history(
12937                        "durable_command_sequence_mismatch",
12938                        sequence,
12939                        &format!("workflow sequence greater than {previous}"),
12940                        &sequence.to_string(),
12941                        "durable commands are not strictly ordered by their recorded workflow sequence",
12942                    ));
12943                }
12944            }
12945            last_new_sequence = Some(sequence);
12946        }
12947        events_by_sequence.entry(sequence).or_default().push(event);
12948    }
12949
12950    let commands: Vec<RecordedCommand> = events_by_sequence
12951        .into_iter()
12952        .map(|(sequence, sequence_events)| {
12953            let activity_events: Vec<_> = sequence_events
12954                .iter()
12955                .copied()
12956                .filter(|event| event.event_type.starts_with("Activity"))
12957                .collect();
12958            let timer_events: Vec<_> = sequence_events
12959                .iter()
12960                .copied()
12961                .filter(|event| event.event_type.starts_with("Timer"))
12962                .collect();
12963            let child_events: Vec<_> = sequence_events
12964                .iter()
12965                .copied()
12966                .filter(|event| {
12967                    event.event_type == "ChildWorkflowScheduled"
12968                        || event.event_type.starts_with("ChildRun")
12969                })
12970                .collect();
12971            let signal_wait_events: Vec<_> = sequence_events
12972                .iter()
12973                .copied()
12974                .filter(|event| is_recorded_signal_wait_event(event))
12975                .collect();
12976            let condition_wait_events: Vec<_> = sequence_events
12977                .iter()
12978                .copied()
12979                .filter(|event| is_recorded_condition_wait_event(event))
12980                .collect();
12981            let search_attribute_events: Vec<_> = sequence_events
12982                .iter()
12983                .copied()
12984                .filter(|event| event.event_type == "SearchAttributesUpserted")
12985                .collect();
12986            let side_effect_events: Vec<_> = sequence_events
12987                .iter()
12988                .copied()
12989                .filter(|event| event.event_type == "SideEffectRecorded")
12990                .collect();
12991            let version_marker_events: Vec<_> = sequence_events
12992                .iter()
12993                .copied()
12994                .filter(|event| event.event_type == "VersionMarkerRecorded")
12995                .collect();
12996            let memo_events: Vec<_> = sequence_events
12997                .iter()
12998                .copied()
12999                .filter(|event| event.event_type == "MemoUpserted")
13000                .collect();
13001
13002            let command_kind_count = usize::from(!activity_events.is_empty())
13003                + usize::from(!timer_events.is_empty())
13004                + usize::from(!child_events.is_empty())
13005                + usize::from(!signal_wait_events.is_empty())
13006                + usize::from(!condition_wait_events.is_empty())
13007                + usize::from(!search_attribute_events.is_empty())
13008                + usize::from(!side_effect_events.is_empty())
13009                + usize::from(!version_marker_events.is_empty())
13010                + usize::from(!memo_events.is_empty());
13011            if command_kind_count > 1 {
13012                let actual = [
13013                    (!activity_events.is_empty()).then_some("activity"),
13014                    (!timer_events.is_empty()).then_some("timer"),
13015                    (!child_events.is_empty()).then_some("child workflow"),
13016                    (!signal_wait_events.is_empty()).then_some("signal wait"),
13017                    (!condition_wait_events.is_empty()).then_some("condition wait"),
13018                    (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13019                    (!side_effect_events.is_empty()).then_some("side effect"),
13020                    (!version_marker_events.is_empty()).then_some("version marker"),
13021                    (!memo_events.is_empty()).then_some("memo upsert"),
13022                ]
13023                .into_iter()
13024                .flatten()
13025                .collect::<Vec<_>>()
13026                .join(" and ");
13027                return Err(invalid_recorded_history(
13028                    "durable_command_sequence_collision",
13029                    sequence,
13030                    "one durable command kind",
13031                    &actual,
13032                    "one workflow sequence records more than one durable command kind",
13033                ));
13034            }
13035
13036            if !activity_events.is_empty() {
13037                let parallel_group_path =
13038                    recorded_parallel_group_path(&activity_events, sequence)?;
13039                let scheduled_count = activity_events
13040                    .iter()
13041                    .filter(|event| event.event_type == "ActivityScheduled")
13042                    .count();
13043                if scheduled_count > 1 {
13044                    return Err(invalid_recorded_history(
13045                        "duplicate_activity_schedule",
13046                        sequence,
13047                        "at most one ActivityScheduled event",
13048                        "multiple ActivityScheduled events",
13049                        "activity history schedules more than one command at one workflow sequence",
13050                    ));
13051                }
13052                let activity_type = activity_events.iter().find_map(|event| {
13053                    event
13054                        .payload
13055                        .get("activity_type")
13056                        .or_else(|| event.payload.get("activity_name"))
13057                        .and_then(Value::as_str)
13058                        .map(str::to_string)
13059                });
13060                if activity_events.iter().filter_map(|event| {
13061                    event
13062                        .payload
13063                        .get("activity_type")
13064                        .or_else(|| event.payload.get("activity_name"))
13065                        .and_then(Value::as_str)
13066                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13067                    return Err(invalid_recorded_history(
13068                        "activity_identity_mismatch",
13069                        sequence,
13070                        activity_type.as_deref().unwrap_or("one activity identity"),
13071                        "conflicting activity identities",
13072                        "activity lifecycle events at one workflow sequence disagree on identity",
13073                    ));
13074                }
13075                let terminal: Vec<_> = activity_events
13076                    .iter()
13077                    .copied()
13078                    .filter(|event| {
13079                        matches!(
13080                            event.event_type.as_str(),
13081                            "ActivityCompleted"
13082                                | "ActivityFailed"
13083                                | "ActivityCancelled"
13084                                | "ActivityTimedOut"
13085                        )
13086                    })
13087                    .collect();
13088                let duplicate_delivery = terminal.first().is_some_and(|first| {
13089                    terminal.iter().all(|event| {
13090                        event.event_type == first.event_type && event.payload == first.payload
13091                    })
13092                });
13093                if terminal.len() > 1 && !duplicate_delivery {
13094                    return Err(invalid_recorded_history(
13095                        "duplicate_activity_terminal_event",
13096                        sequence,
13097                        "at most one terminal activity event",
13098                        "multiple terminal activity events",
13099                        "activity history settles one command more than once",
13100                    ));
13101                }
13102                let outcome = terminal
13103                    .first()
13104                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13105                    .transpose()?;
13106                let options = activity_events
13107                    .iter()
13108                    .find(|event| event.event_type == "ActivityScheduled")
13109                    .and_then(|event| event.payload.get("activity"))
13110                    .and_then(Value::as_object)
13111                    .map(|activity| RecordedActivityOptions {
13112                        task_queue: recorded_optional_string(activity, "queue"),
13113                        execution_mode: recorded_optional_string(activity, "execution_mode"),
13114                        retry_policy: recorded_activity_retry_snapshot(
13115                            activity.get("retry_policy"),
13116                        ),
13117                    });
13118                return Ok(RecordedCommand::Activity {
13119                    sequence,
13120                    activity_type,
13121                    options,
13122                    outcome,
13123                    parallel_group_path,
13124                });
13125            }
13126
13127            if !child_events.is_empty() {
13128                let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13129                let scheduled: Vec<_> = child_events
13130                    .iter()
13131                    .copied()
13132                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
13133                    .collect();
13134                if scheduled.len() != 1 {
13135                    return Err(invalid_recorded_history(
13136                        "child_workflow_schedule_missing_or_duplicate",
13137                        sequence,
13138                        "one ChildWorkflowScheduled event",
13139                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13140                        "child workflow replay requires exactly one recorded schedule event",
13141                    ));
13142                }
13143                let workflow_type = child_events.iter().find_map(|event| {
13144                    event
13145                        .payload
13146                        .get("child_workflow_type")
13147                        .or_else(|| event.payload.get("workflow_type"))
13148                        .and_then(Value::as_str)
13149                        .filter(|value| !value.is_empty())
13150                        .map(str::to_string)
13151                });
13152                if child_events
13153                    .iter()
13154                    .filter_map(|event| {
13155                        event
13156                            .payload
13157                            .get("child_workflow_type")
13158                            .or_else(|| event.payload.get("workflow_type"))
13159                            .and_then(Value::as_str)
13160                    })
13161                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
13162                {
13163                    return Err(invalid_recorded_history(
13164                        "child_workflow_identity_mismatch",
13165                        sequence,
13166                        workflow_type
13167                            .as_deref()
13168                            .unwrap_or("one child workflow type"),
13169                        "conflicting child workflow types",
13170                        "child workflow lifecycle events at one sequence disagree on type",
13171                    ));
13172                }
13173                let mut outcomes = child_workflow_outcomes(
13174                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13175                    fallback_codec,
13176                    parent.clone(),
13177                )?;
13178                let terminal_events = child_events
13179                    .iter()
13180                    .copied()
13181                    .filter(|event| event.event_type.starts_with("ChildRun"))
13182                    .collect::<Vec<_>>();
13183                let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13184                    terminal_events.iter().all(|event| {
13185                        event.event_type == first.event_type && event.payload == first.payload
13186                    })
13187                });
13188                if outcomes.len() > 1 && !duplicate_delivery {
13189                    return Err(invalid_recorded_history(
13190                        "duplicate_child_workflow_terminal_event",
13191                        sequence,
13192                        "at most one terminal child event",
13193                        "multiple terminal child events",
13194                        "child workflow history settles one command more than once",
13195                    ));
13196                }
13197                return Ok(RecordedCommand::ChildWorkflow {
13198                    sequence,
13199                    workflow_type,
13200                    outcome: outcomes.pop(),
13201                    parallel_group_path,
13202                });
13203            }
13204
13205            if !signal_wait_events.is_empty() {
13206                let opened: Vec<_> = signal_wait_events
13207                    .iter()
13208                    .copied()
13209                    .filter(|event| event.event_type == "SignalWaitOpened")
13210                    .collect();
13211                if opened.len() != 1 {
13212                    return Err(invalid_recorded_history(
13213                        "signal_wait_open_missing_or_duplicate",
13214                        sequence,
13215                        "one SignalWaitOpened event",
13216                        &format!("{} SignalWaitOpened events", opened.len()),
13217                        "signal replay requires exactly one canonical wait-open event",
13218                    ));
13219                }
13220
13221                let applied: Vec<_> = signal_wait_events
13222                    .iter()
13223                    .copied()
13224                    .filter(|event| event.event_type == "SignalApplied")
13225                    .collect();
13226                if applied.len() > 1 {
13227                    return Err(invalid_recorded_history(
13228                        "duplicate_signal_wait_apply",
13229                        sequence,
13230                        "at most one SignalApplied event",
13231                        "multiple SignalApplied events",
13232                        "signal history applies one durable wait more than once",
13233                    ));
13234                }
13235
13236                let signal_names = signal_wait_events
13237                    .iter()
13238                    .map(|event| required_signal_wait_name(event, sequence))
13239                    .collect::<Result<Vec<_>>>()?;
13240                let signal_name = signal_names
13241                    .first()
13242                    .expect("signal wait events are not empty")
13243                    .clone();
13244                if signal_names.iter().any(|candidate| candidate != &signal_name) {
13245                    return Err(invalid_recorded_history(
13246                        "signal_wait_identity_mismatch",
13247                        sequence,
13248                        &signal_name,
13249                        "conflicting signal names",
13250                        "signal wait lifecycle events at one workflow sequence disagree on identity",
13251                    ));
13252                }
13253                let value = applied
13254                    .first()
13255                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
13256                    .transpose()?;
13257                return Ok(RecordedCommand::SignalWait {
13258                    sequence,
13259                    signal_name,
13260                    value,
13261                    parallel_group_path: recorded_parallel_group_path(
13262                        &signal_wait_events,
13263                        sequence,
13264                    )?,
13265                });
13266            }
13267
13268            if !condition_wait_events.is_empty() {
13269                return recorded_condition_wait(
13270                    sequence,
13271                    &condition_wait_events,
13272                    events,
13273                );
13274            }
13275
13276            if !search_attribute_events.is_empty() {
13277                if search_attribute_events.len() != 1 {
13278                    return Err(invalid_recorded_history(
13279                        "duplicate_search_attribute_update",
13280                        sequence,
13281                        "one SearchAttributesUpserted event",
13282                        &format!(
13283                            "{} SearchAttributesUpserted events",
13284                            search_attribute_events.len()
13285                        ),
13286                        "search-attribute history records one workflow command more than once",
13287                    ));
13288                }
13289                let payload = &search_attribute_events[0].payload;
13290                let attributes = payload
13291                    .get("attributes")
13292                    .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13293                    .cloned()
13294                    .ok_or_else(|| {
13295                        invalid_recorded_history(
13296                            "search_attribute_update_missing",
13297                            sequence,
13298                            "non-empty attributes object",
13299                            "missing or invalid attributes",
13300                            "search-attribute history is missing its recorded mutation",
13301                        )
13302                    })?;
13303                let attribute_types =
13304                    recorded_search_attribute_types(payload, &attributes, sequence)?;
13305                return Ok(RecordedCommand::SearchAttributes {
13306                    sequence,
13307                    attributes,
13308                    attribute_types,
13309                });
13310            }
13311
13312            if !side_effect_events.is_empty() {
13313                if side_effect_events.len() != 1 {
13314                    return Err(invalid_recorded_history(
13315                        "duplicate_side_effect_record",
13316                        sequence,
13317                        "one SideEffectRecorded event",
13318                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
13319                        "side-effect history records one workflow command more than once",
13320                    ));
13321                }
13322                let event = side_effect_events[0];
13323                let result = event.payload.get("result").ok_or_else(|| {
13324                    invalid_recorded_history(
13325                        "side_effect_result_missing",
13326                        sequence,
13327                        "recorded result payload",
13328                        "missing result",
13329                        "side-effect history is missing its recorded value",
13330                    )
13331                })?;
13332                let has_published_envelope = result.as_str().is_some()
13333                    || result.as_object().is_some_and(|envelope| {
13334                        envelope.get("codec").and_then(Value::as_str).is_some()
13335                            && envelope.get("blob").and_then(Value::as_str).is_some()
13336                    });
13337                if !has_published_envelope {
13338                    return Err(invalid_recorded_history(
13339                        "side_effect_payload_malformed",
13340                        sequence,
13341                        "payload blob or {codec, blob} envelope",
13342                        &result.to_string(),
13343                        "side-effect history result does not use a published payload envelope",
13344                    ));
13345                }
13346                let codec = event
13347                    .payload
13348                    .get("payload_codec")
13349                    .and_then(Value::as_str)
13350                    .unwrap_or(fallback_codec);
13351                let value = decode_wire_avro_value(result, codec).map_err(|error| {
13352                    if error.to_string().contains("unsupported_payload_codec") {
13353                        return error;
13354                    }
13355
13356                    invalid_recorded_history(
13357                        "side_effect_payload_incompatible",
13358                        sequence,
13359                        &format!("valid {codec} payload envelope"),
13360                        &error.to_string(),
13361                        "side-effect history payload cannot be decoded with its recorded codec",
13362                    )
13363                })?;
13364                return Ok(RecordedCommand::SideEffect { sequence, value });
13365            }
13366
13367            if !version_marker_events.is_empty() {
13368                if version_marker_events.len() != 1 {
13369                    return Err(invalid_recorded_history(
13370                        "duplicate_version_marker_record",
13371                        sequence,
13372                        "one VersionMarkerRecorded event",
13373                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13374                        "version-marker history records one workflow command more than once",
13375                    ));
13376                }
13377                let payload = &version_marker_events[0].payload;
13378                let change_id = payload
13379                    .get("change_id")
13380                    .and_then(Value::as_str)
13381                    .filter(|value| !value.is_empty())
13382                    .map(str::to_string)
13383                    .ok_or_else(|| {
13384                        invalid_recorded_history(
13385                            "version_marker_field_missing",
13386                            sequence,
13387                            "non-empty change_id",
13388                            "missing or invalid change_id",
13389                            "version-marker history is missing its stable change ID",
13390                        )
13391                    })?;
13392                let version = required_version_i32(payload, "version", sequence)?;
13393                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13394                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13395                if min_supported > max_supported || version < min_supported || version > max_supported {
13396                    return Err(invalid_recorded_history(
13397                        "version_marker_history_range_invalid",
13398                        sequence,
13399                        "min_supported <= version <= max_supported",
13400                        &format!("{min_supported} <= {version} <= {max_supported}"),
13401                        "recorded version marker contains an internally incompatible range",
13402                    ));
13403                }
13404                return Ok(RecordedCommand::VersionMarker {
13405                    sequence,
13406                    change_id,
13407                    version,
13408                });
13409            }
13410
13411            if !memo_events.is_empty() {
13412                if memo_events.len() != 1 {
13413                    return Err(invalid_recorded_history(
13414                        "duplicate_memo_upsert_record",
13415                        sequence,
13416                        "one MemoUpserted event",
13417                        &format!("{} MemoUpserted events", memo_events.len()),
13418                        "memo history records one workflow update more than once",
13419                    ));
13420                }
13421                let payload = &memo_events[0].payload;
13422                let entries = payload.get("entries").cloned().ok_or_else(|| {
13423                    invalid_recorded_history(
13424                        "memo_entries_missing",
13425                        sequence,
13426                        "memo entries object",
13427                        "missing entries",
13428                        "MemoUpserted history is missing replay identity entries",
13429                    )
13430                })?;
13431                let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13432                    invalid_recorded_history(
13433                        "memo_entries_invalid",
13434                        sequence,
13435                        "valid canonical memo entries",
13436                        &error.to_string(),
13437                        "MemoUpserted history contains invalid replay identity entries",
13438                    )
13439                })?;
13440                let merged = payload.get("merged").cloned().ok_or_else(|| {
13441                    invalid_recorded_history(
13442                        "memo_merged_projection_missing",
13443                        sequence,
13444                        "merged memo projection",
13445                        "missing merged",
13446                        "MemoUpserted history is missing its merged projection",
13447                    )
13448                })?;
13449                decode_memo_history_map(&merged, false).map_err(|error| {
13450                    invalid_recorded_history(
13451                        "memo_merged_projection_invalid",
13452                        sequence,
13453                        "valid merged memo projection",
13454                        &error.to_string(),
13455                        "MemoUpserted history contains an invalid merged projection",
13456                    )
13457                })?;
13458
13459                return Ok(RecordedCommand::Memo { sequence, entries });
13460            }
13461            let scheduled: Vec<_> = timer_events
13462                .iter()
13463                .copied()
13464                .filter(|event| event.event_type == "TimerScheduled")
13465                .collect();
13466            let fired: Vec<_> = timer_events
13467                .iter()
13468                .copied()
13469                .filter(|event| event.event_type == "TimerFired")
13470                .collect();
13471            if scheduled.len() != 1 {
13472                return Err(invalid_recorded_history(
13473                    "timer_schedule_missing_or_duplicate",
13474                    sequence,
13475                    "one TimerScheduled event",
13476                    &format!("{} TimerScheduled events", scheduled.len()),
13477                    "timer replay requires exactly one recorded schedule event",
13478                ));
13479            }
13480            if fired.len() > 1 {
13481                return Err(invalid_recorded_history(
13482                    "duplicate_timer_fire",
13483                    sequence,
13484                    "at most one TimerFired event",
13485                    "multiple TimerFired events",
13486                    "timer history contains more than one fire event for a workflow sequence",
13487                ));
13488            }
13489
13490            let scheduled = scheduled[0];
13491            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13492            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13493            if let Some(fired) = fired.first() {
13494                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13495                if fired_timer_id != timer_id {
13496                    return Err(invalid_recorded_history(
13497                        "timer_identity_mismatch",
13498                        sequence,
13499                        &timer_id,
13500                        &fired_timer_id,
13501                        "TimerFired does not correspond to the recorded TimerScheduled event",
13502                    ));
13503                }
13504                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13505                if fired_delay != delay_seconds {
13506                    return Err(invalid_recorded_history(
13507                        "timer_history_delay_mismatch",
13508                        sequence,
13509                        &delay_seconds.to_string(),
13510                        &fired_delay.to_string(),
13511                        "TimerScheduled and TimerFired record different delays",
13512                    ));
13513                }
13514            }
13515
13516            Ok(RecordedCommand::Timer {
13517                sequence,
13518                delay_seconds,
13519                fired: !fired.is_empty(),
13520                parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13521            })
13522        })
13523        .collect::<Result<_>>()?;
13524
13525    let mut marker_sequences = HashMap::new();
13526    for command in &commands {
13527        if let RecordedCommand::VersionMarker {
13528            sequence,
13529            change_id,
13530            ..
13531        } = command
13532        {
13533            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13534                return Err(invalid_recorded_history(
13535                    "duplicate_version_marker",
13536                    *sequence,
13537                    &format!("one marker for change ID {change_id:?}"),
13538                    &format!("markers at sequences {first_sequence} and {sequence}"),
13539                    "workflow history contains duplicate markers for one stable change ID",
13540                ));
13541            }
13542        }
13543    }
13544
13545    Ok(commands)
13546}
13547
13548fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13549    payload
13550        .get(field)
13551        .and_then(Value::as_i64)
13552        .and_then(|value| i32::try_from(value).ok())
13553        .ok_or_else(|| {
13554            invalid_recorded_history(
13555                "version_marker_field_missing",
13556                sequence,
13557                &format!("integer {field}"),
13558                "missing or out-of-range integer",
13559                "version-marker history is missing a required integer field",
13560            )
13561        })
13562}
13563
13564fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13565    event
13566        .payload
13567        .get("sequence")
13568        .or_else(|| event.payload.get("workflow_sequence"))
13569        .or_else(|| event.raw.get("sequence"))
13570        .or_else(|| event.raw.get("workflow_sequence"))
13571        .and_then(value_as_u64)
13572}
13573
13574fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13575    matches!(
13576        event
13577            .payload
13578            .get("timer_kind")
13579            .or_else(|| event.raw.get("timer_kind"))
13580            .and_then(Value::as_str),
13581        Some("condition_timeout" | "signal_timeout")
13582    )
13583}
13584
13585fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13586    matches!(
13587        event.event_type.as_str(),
13588        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13589    )
13590}
13591
13592fn recorded_condition_wait(
13593    sequence: u64,
13594    condition_events: &[&HistoryEvent],
13595    all_events: &[HistoryEvent],
13596) -> Result<RecordedCommand> {
13597    let opened = condition_events
13598        .iter()
13599        .copied()
13600        .filter(|event| event.event_type == "ConditionWaitOpened")
13601        .collect::<Vec<_>>();
13602    if opened.len() != 1 {
13603        return Err(invalid_recorded_history(
13604            "condition_wait_open_missing_or_duplicate",
13605            sequence,
13606            "one ConditionWaitOpened event",
13607            &format!("{} ConditionWaitOpened events", opened.len()),
13608            "condition replay requires exactly one canonical wait-open event",
13609        ));
13610    }
13611    let terminal = condition_events
13612        .iter()
13613        .copied()
13614        .filter(|event| {
13615            matches!(
13616                event.event_type.as_str(),
13617                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13618            )
13619        })
13620        .collect::<Vec<_>>();
13621    if terminal.len() > 1 {
13622        return Err(invalid_recorded_history(
13623            "duplicate_condition_wait_terminal_event",
13624            sequence,
13625            "at most one condition terminal event",
13626            "multiple condition terminal events",
13627            "condition history settles one durable wait more than once",
13628        ));
13629    }
13630
13631    let opened = opened[0];
13632    let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13633    let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13634    for event in condition_events
13635        .iter()
13636        .copied()
13637        .filter(|event| !std::ptr::eq(*event, opened))
13638    {
13639        let event_wait_id = required_condition_wait_id(event, sequence)?;
13640        if event_wait_id != condition_wait_id {
13641            return Err(invalid_recorded_history(
13642                "condition_wait_id_mismatch",
13643                sequence,
13644                &condition_wait_id,
13645                &event_wait_id,
13646                "condition lifecycle events at one sequence disagree on wait identity",
13647            ));
13648        }
13649        let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13650        if event_occurrence_id != occurrence_id {
13651            return Err(invalid_recorded_history(
13652                "condition_wait_occurrence_history_mismatch",
13653                sequence,
13654                &occurrence_id,
13655                &event_occurrence_id,
13656                "condition lifecycle events at one sequence disagree on authored occurrence identity",
13657            ));
13658        }
13659    }
13660
13661    let condition_key = optional_non_empty_history_string(opened, "condition_key");
13662    let predicate_identity = opened
13663        .payload
13664        .get("condition_definition_fingerprint")
13665        .and_then(Value::as_str)
13666        .filter(|value| !value.is_empty())
13667        .map(str::to_string)
13668        .ok_or_else(|| {
13669            invalid_recorded_history(
13670                "condition_wait_predicate_fingerprint_missing",
13671                sequence,
13672                "non-empty condition_definition_fingerprint",
13673                &opened.event_type,
13674                "canonical condition history is missing its predicate identity",
13675            )
13676        })?;
13677    let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13678    for event in condition_events
13679        .iter()
13680        .copied()
13681        .filter(|event| !std::ptr::eq(*event, opened))
13682    {
13683        for (field, opened_value) in [
13684            ("condition_key", condition_key.as_deref()),
13685            (
13686                "condition_definition_fingerprint",
13687                Some(predicate_identity.as_str()),
13688            ),
13689        ] {
13690            if let Some(value) = optional_non_empty_history_string(event, field) {
13691                if opened_value.is_some_and(|opened_value| opened_value != value) {
13692                    return Err(invalid_recorded_history(
13693                        "condition_wait_definition_history_mismatch",
13694                        sequence,
13695                        opened_value.unwrap_or_default(),
13696                        &value,
13697                        "condition lifecycle events disagree on the recorded definition",
13698                    ));
13699                }
13700            }
13701        }
13702        if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13703            if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13704                return Err(invalid_recorded_history(
13705                    "condition_wait_definition_history_mismatch",
13706                    sequence,
13707                    &format!("{}s", timeout_seconds.unwrap_or_default()),
13708                    &format!("{event_timeout}s"),
13709                    "condition lifecycle events disagree on the recorded timeout",
13710                ));
13711            }
13712        }
13713    }
13714
13715    let timeout_timer_events = all_events
13716        .iter()
13717        .filter(|event| {
13718            matches!(
13719                event.event_type.as_str(),
13720                "TimerScheduled" | "TimerCancelled" | "TimerFired"
13721            ) && event.payload.get("timer_kind").and_then(Value::as_str)
13722                == Some("condition_timeout")
13723                && event
13724                    .payload
13725                    .get("condition_wait_id")
13726                    .and_then(Value::as_str)
13727                    == Some(condition_wait_id.as_str())
13728        })
13729        .collect::<Vec<_>>();
13730    let scheduled = timeout_timer_events
13731        .iter()
13732        .copied()
13733        .filter(|event| event.event_type == "TimerScheduled")
13734        .collect::<Vec<_>>();
13735    let fired = timeout_timer_events
13736        .iter()
13737        .copied()
13738        .filter(|event| event.event_type == "TimerFired")
13739        .collect::<Vec<_>>();
13740    if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13741        return Err(invalid_recorded_history(
13742            "condition_wait_timeout_history_invalid",
13743            sequence,
13744            "one timeout schedule and at most one fire",
13745            &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13746            "condition timeout history has a missing or duplicate lifecycle event",
13747        ));
13748    }
13749    if let Some(scheduled) = scheduled.first() {
13750        let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13751        let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13752        if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13753            return Err(invalid_recorded_history(
13754                "condition_wait_timeout_delay_mismatch",
13755                sequence,
13756                &format!("{}s", timeout_seconds.unwrap_or_default()),
13757                &format!("{delay_seconds}s"),
13758                "condition timeout timer differs from the wait definition",
13759            ));
13760        }
13761        if let Some(fired) = fired.first() {
13762            let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13763            let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13764            if fired_timer_id != timer_id || fired_delay != delay_seconds {
13765                return Err(invalid_recorded_history(
13766                    "condition_wait_timeout_identity_mismatch",
13767                    sequence,
13768                    &format!("{timer_id}:{delay_seconds}s"),
13769                    &format!("{fired_timer_id}:{fired_delay}s"),
13770                    "condition timeout fire does not match its durable schedule",
13771                ));
13772            }
13773        }
13774    }
13775
13776    let result = terminal.first().map(|event| {
13777        if event.event_type == "ConditionWaitTimedOut" {
13778            ConditionWaitResult::TimedOut
13779        } else {
13780            ConditionWaitResult::Satisfied
13781        }
13782    });
13783    let result = if !fired.is_empty() {
13784        if result == Some(ConditionWaitResult::Satisfied) {
13785            return Err(invalid_recorded_history(
13786                "condition_wait_terminal_conflict",
13787                sequence,
13788                "one satisfied or timed-out outcome",
13789                "satisfied event and fired timeout",
13790                "condition history records conflicting terminal outcomes",
13791            ));
13792        }
13793        Some(ConditionWaitResult::TimedOut)
13794    } else {
13795        result
13796    };
13797
13798    Ok(RecordedCommand::ConditionWait {
13799        sequence,
13800        occurrence_id,
13801        condition_key,
13802        predicate_identity,
13803        timeout_seconds,
13804        result,
13805        parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13806    })
13807}
13808
13809fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13810    event
13811        .payload
13812        .get("condition_wait_occurrence_id")
13813        .and_then(Value::as_str)
13814        .filter(|value| !value.is_empty())
13815        .map(str::to_string)
13816        .ok_or_else(|| {
13817            invalid_recorded_history(
13818                "condition_wait_occurrence_id_missing",
13819                sequence,
13820                "non-empty condition_wait_occurrence_id",
13821                &event.event_type,
13822                "condition history is missing authored occurrence identity",
13823            )
13824        })
13825}
13826
13827fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13828    event
13829        .payload
13830        .get("condition_wait_id")
13831        .and_then(Value::as_str)
13832        .filter(|value| !value.is_empty())
13833        .map(str::to_string)
13834        .ok_or_else(|| {
13835            invalid_recorded_history(
13836                "condition_wait_id_missing",
13837                sequence,
13838                "non-empty condition_wait_id",
13839                &event.event_type,
13840                "canonical condition history is missing its durable wait identity",
13841            )
13842        })
13843}
13844
13845fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13846    event
13847        .payload
13848        .get(field)
13849        .and_then(Value::as_str)
13850        .filter(|value| !value.is_empty())
13851        .map(str::to_string)
13852}
13853
13854fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13855    match event.payload.get(field) {
13856        None | Some(Value::Null) => Ok(None),
13857        Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13858            invalid_recorded_history(
13859                "condition_wait_definition_invalid",
13860                sequence,
13861                &format!("non-negative integer {field}"),
13862                &value.to_string(),
13863                "condition history contains an invalid numeric definition field",
13864            )
13865        }),
13866    }
13867}
13868
13869fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13870    event
13871        .payload
13872        .get("signal_name")
13873        .or_else(|| event.raw.get("signal_name"))
13874        .and_then(Value::as_str)
13875        .filter(|value| !value.is_empty())
13876        .map(str::to_string)
13877        .ok_or_else(|| {
13878            invalid_recorded_history(
13879                "signal_wait_name_missing",
13880                sequence,
13881                "non-empty signal_name",
13882                &event.event_type,
13883                "canonical signal-wait history is missing its signal identity",
13884            )
13885        })
13886}
13887
13888fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13889    matches!(
13890        event.event_type.as_str(),
13891        "SignalWaitOpened" | "SignalApplied"
13892    )
13893}
13894
13895fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
13896    event
13897        .payload
13898        .get(field)
13899        .and_then(Value::as_str)
13900        .filter(|value| !value.is_empty())
13901        .map(str::to_string)
13902        .ok_or_else(|| {
13903            invalid_recorded_history(
13904                "timer_history_field_missing",
13905                sequence,
13906                field,
13907                &event.event_type,
13908                "timer history is missing a required identity field",
13909            )
13910        })
13911}
13912
13913fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
13914    event
13915        .payload
13916        .get(field)
13917        .and_then(value_as_u64)
13918        .ok_or_else(|| {
13919            invalid_recorded_history(
13920                "timer_history_field_missing",
13921                sequence,
13922                field,
13923                &event.event_type,
13924                "timer history is missing a required numeric field",
13925            )
13926        })
13927}
13928
13929fn recorded_search_attribute_types(
13930    payload: &Value,
13931    attributes: &Value,
13932    sequence: u64,
13933) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
13934    let Some(raw_types) = payload.get("attribute_types") else {
13935        // This is the explicit compatibility rule for histories recorded
13936        // before typed identity was persisted. Values still constrain replay;
13937        // the unknown type snapshot does not assert a typed match.
13938        return Ok(RecordedSnapshotValue::Unknown);
13939    };
13940    let Some(raw_types) = raw_types.as_object() else {
13941        return Err(invalid_recorded_history(
13942            "search_attribute_types_malformed",
13943            sequence,
13944            "canonical attribute type map",
13945            &raw_types.to_string(),
13946            "search-attribute history contains malformed type identity",
13947        ));
13948    };
13949    let attribute_keys = attributes
13950        .as_object()
13951        .expect("recorded search attributes were validated as an object");
13952    let mut types = BTreeMap::new();
13953    for (key, value) in raw_types {
13954        let Some(attribute_type) = value.as_str() else {
13955            return Err(invalid_recorded_history(
13956                "search_attribute_types_malformed",
13957                sequence,
13958                "canonical string type name",
13959                &value.to_string(),
13960                "search-attribute history contains a non-string type identity",
13961            ));
13962        };
13963        if !attribute_keys.contains_key(key)
13964            || !matches!(
13965                attribute_type,
13966                "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
13967            )
13968        {
13969            return Err(invalid_recorded_history(
13970                "search_attribute_types_malformed",
13971                sequence,
13972                "canonical types for keys present in attributes",
13973                &format!("{key}:{attribute_type}"),
13974                "search-attribute history contains unsupported or orphaned type identity",
13975            ));
13976        }
13977        types.insert(key.clone(), attribute_type.to_string());
13978    }
13979    Ok(RecordedSnapshotValue::Known(types))
13980}
13981
13982fn invalid_recorded_history(
13983    reason: &str,
13984    sequence: u64,
13985    expected: &str,
13986    actual: &str,
13987    message: &str,
13988) -> Error {
13989    Error::NonDeterministicReplay(ReplayFailure::new(
13990        reason,
13991        Some(sequence),
13992        Some(expected.to_string()),
13993        Some(actual.to_string()),
13994        message,
13995    ))
13996}
13997
13998type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
13999
14000fn activity_outcome(
14001    event: &HistoryEvent,
14002    fallback_codec: &str,
14003    recorded_activity_type: Option<String>,
14004) -> Result<ActivityOutcome> {
14005    if event.event_type == "ActivityCompleted" {
14006        let codec = event
14007            .payload
14008            .get("payload_codec")
14009            .and_then(Value::as_str)
14010            .unwrap_or(fallback_codec);
14011        return Ok(Ok(decode_wire_avro_value(
14012            event.payload.get("result").unwrap_or(&Value::Null),
14013            codec,
14014        )?));
14015    }
14016
14017    let payload = &event.payload;
14018    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14019        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14020        "ActivityCancelled" => (
14021            ActivityFailureKind::Cancelled,
14022            "cancelled",
14023            "activity was cancelled",
14024        ),
14025        "ActivityTimedOut" => (
14026            ActivityFailureKind::TimedOut,
14027            "timeout",
14028            "activity timed out",
14029        ),
14030        _ => unreachable!("activity_outcome is called only for terminal activity events"),
14031    };
14032    let exception = payload
14033        .get("exception")
14034        .filter(|value| !value.is_null())
14035        .cloned();
14036    let failure_category = payload_string(payload, "failure_category");
14037    let timeout_kind = payload_string(payload, "timeout_kind");
14038    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14039        ActivityFailureKind::Failed => failure_category
14040            .clone()
14041            .unwrap_or_else(|| fallback_reason.to_string()),
14042        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14043        ActivityFailureKind::TimedOut => timeout_kind
14044            .clone()
14045            .unwrap_or_else(|| fallback_reason.to_string()),
14046    });
14047    let message = payload_string(payload, "message")
14048        .or_else(|| {
14049            exception
14050                .as_ref()
14051                .and_then(|value| payload_string(value, "message"))
14052        })
14053        .unwrap_or_else(|| fallback_message.to_string());
14054
14055    Ok(Err(ActivityFailure {
14056        kind,
14057        reason,
14058        message,
14059        activity_execution_id: payload_string(payload, "activity_execution_id"),
14060        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14061        activity_type: payload_string(payload, "activity_type")
14062            .or_else(|| payload_string(payload, "activity_name"))
14063            .or(recorded_activity_type),
14064        activity_class: payload_string(payload, "activity_class"),
14065        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14066        failure_id: payload_string(payload, "failure_id"),
14067        failure_category,
14068        timeout_kind,
14069        non_retryable: payload
14070            .get("non_retryable")
14071            .and_then(Value::as_bool)
14072            .unwrap_or(false),
14073        exception_type: payload_string(payload, "exception_type").or_else(|| {
14074            exception
14075                .as_ref()
14076                .and_then(|value| payload_string(value, "type"))
14077        }),
14078        exception_class: payload_string(payload, "exception_class").or_else(|| {
14079            exception
14080                .as_ref()
14081                .and_then(|value| payload_string(value, "class"))
14082        }),
14083        code: payload
14084            .get("code")
14085            .filter(|value| !value.is_null())
14086            .cloned(),
14087        exception,
14088    }))
14089}
14090
14091type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14092
14093fn child_workflow_outcomes(
14094    events: &[HistoryEvent],
14095    fallback_codec: &str,
14096    parent: WorkflowIdentity,
14097) -> Result<Vec<ChildWorkflowOutcome>> {
14098    let mut outcomes = Vec::new();
14099
14100    for event in events {
14101        let kind = match event.event_type.as_str() {
14102            "ChildRunCompleted" => None,
14103            "ChildRunFailed" => Some((
14104                ChildWorkflowFailureKind::Failed,
14105                "child_workflow",
14106                "child workflow failed",
14107            )),
14108            "ChildRunCancelled" => Some((
14109                ChildWorkflowFailureKind::Cancelled,
14110                "cancelled",
14111                "child workflow was cancelled",
14112            )),
14113            "ChildRunTerminated" => Some((
14114                ChildWorkflowFailureKind::Terminated,
14115                "terminated",
14116                "child workflow was terminated",
14117            )),
14118            _ => continue,
14119        };
14120        let payload = &event.payload;
14121        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14122        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14123        let child_workflow_type = payload_string(payload, "child_workflow_type");
14124
14125        if let Some((kind, reason, fallback_message)) = kind {
14126            let exception = payload
14127                .get("exception")
14128                .filter(|value| !value.is_null())
14129                .cloned();
14130            let message = payload_string(payload, "message")
14131                .or_else(|| {
14132                    exception
14133                        .as_ref()
14134                        .and_then(|value| payload_string(value, "message"))
14135                })
14136                .unwrap_or_else(|| fallback_message.to_string());
14137            let exception_type = payload_string(payload, "exception_type").or_else(|| {
14138                exception
14139                    .as_ref()
14140                    .and_then(|value| payload_string(value, "type"))
14141            });
14142            let exception_class = payload_string(payload, "exception_class").or_else(|| {
14143                exception
14144                    .as_ref()
14145                    .and_then(|value| payload_string(value, "class"))
14146            });
14147            outcomes.push(Err(ChildWorkflowFailure {
14148                kind,
14149                reason: reason.to_string(),
14150                message,
14151                parent_workflow_id: parent.workflow_id.clone(),
14152                parent_workflow_run_id: parent.run_id.clone(),
14153                child_workflow_id,
14154                child_workflow_run_id,
14155                child_workflow_type,
14156                failure_id: payload_string(payload, "failure_id"),
14157                failure_category: payload_string(payload, "failure_category"),
14158                exception_type,
14159                exception_class,
14160                non_retryable: payload
14161                    .get("non_retryable")
14162                    .and_then(Value::as_bool)
14163                    .unwrap_or(false),
14164                code: payload
14165                    .get("code")
14166                    .filter(|value| !value.is_null())
14167                    .cloned(),
14168                exception,
14169            }));
14170            continue;
14171        }
14172
14173        let codec = payload
14174            .get("payload_codec")
14175            .and_then(Value::as_str)
14176            .unwrap_or(fallback_codec);
14177        let result = payload
14178            .get("result")
14179            .or_else(|| payload.get("output"))
14180            .unwrap_or(&Value::Null);
14181        outcomes.push(Ok(ChildWorkflowAvroResult {
14182            parent: parent.clone(),
14183            child: WorkflowIdentity {
14184                workflow_id: child_workflow_id,
14185                run_id: child_workflow_run_id,
14186            },
14187            child_workflow_type,
14188            result: decode_wire_avro_value(result, codec)?,
14189        }));
14190    }
14191
14192    Ok(outcomes)
14193}
14194
14195fn payload_string(payload: &Value, key: &str) -> Option<String> {
14196    payload
14197        .get(key)
14198        .and_then(Value::as_str)
14199        .filter(|value| !value.is_empty())
14200        .map(str::to_string)
14201}
14202
14203fn workflow_failure_command(error: &Error) -> Value {
14204    let (exception_type, exception_class, properties) = match error {
14205        Error::ActivityFailed(failure) => (
14206            match failure.kind {
14207                ActivityFailureKind::Failed => "ActivityFailed",
14208                ActivityFailureKind::Cancelled => "ActivityCancelled",
14209                ActivityFailureKind::TimedOut => "ActivityTimedOut",
14210            },
14211            "durable_workflow::ActivityFailure",
14212            json!({
14213                "reason": failure.reason,
14214                "activity_execution_id": failure.activity_execution_id,
14215                "activity_attempt_id": failure.activity_attempt_id,
14216                "activity_type": failure.activity_type,
14217                "activity_class": failure.activity_class,
14218                "attempt_number": failure.attempt_number,
14219                "failure_id": failure.failure_id,
14220                "failure_category": failure.failure_category,
14221                "timeout_kind": failure.timeout_kind,
14222                "activity_non_retryable": failure.non_retryable,
14223                "activity_exception_type": failure.exception_type,
14224                "activity_exception_class": failure.exception_class,
14225                "activity_code": failure.code,
14226                "activity_exception": failure.exception,
14227            }),
14228        ),
14229        Error::ChildWorkflowFailed(failure) => (
14230            match failure.kind {
14231                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14232                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14233                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14234            },
14235            "durable_workflow::ChildWorkflowFailure",
14236            json!({
14237                "reason": failure.reason,
14238                "parent_workflow_id": failure.parent_workflow_id,
14239                "parent_workflow_run_id": failure.parent_workflow_run_id,
14240                "child_workflow_id": failure.child_workflow_id,
14241                "child_workflow_run_id": failure.child_workflow_run_id,
14242                "child_workflow_type": failure.child_workflow_type,
14243                "failure_id": failure.failure_id,
14244                "failure_category": failure.failure_category,
14245                "child_exception_type": failure.exception_type,
14246                "child_exception_class": failure.exception_class,
14247                "child_non_retryable": failure.non_retryable,
14248                "child_code": failure.code,
14249                "child_exception": failure.exception,
14250            }),
14251        ),
14252        Error::ParallelFailed(failure) => (
14253            "ParallelFailed",
14254            "durable_workflow::ParallelFailure",
14255            json!({
14256                "parallel_group_id": failure.group_id,
14257                "parallel_member_path": failure.member_path,
14258                "parallel_group_path": failure.group_path,
14259                "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14260                "cause_type": workflow_error_type(&failure.cause),
14261                "cause_message": failure.cause.to_string(),
14262            }),
14263        ),
14264        Error::SagaCompensationFailed(failure) => (
14265            "SagaCompensationFailed",
14266            "durable_workflow::SagaCompensationFailure",
14267            json!({
14268                "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14269                "initiating_failure_message": failure.initiating_failure.to_string(),
14270                "compensation_activity_type": failure.compensation_activity_type,
14271                "compensation_registration_order": failure.compensation_registration_order,
14272                "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14273                "compensation_failure_message": failure.compensation_failure.to_string(),
14274            }),
14275        ),
14276        Error::WorkflowCancellationRequested(_) => (
14277            "WorkflowCancellationRequested",
14278            "durable_workflow::WorkflowCancellationRequested",
14279            json!({"reason": "cancelled"}),
14280        ),
14281        Error::NonDeterministicReplay(_) => (
14282            "NonDeterministicReplay",
14283            "durable_workflow::Error",
14284            Value::Null,
14285        ),
14286        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14287    };
14288    let non_retryable = match error {
14289        Error::ActivityFailed(failure) => failure.non_retryable,
14290        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14291        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14292        Error::SagaCompensationFailed(failure) => {
14293            workflow_error_non_retryable(&failure.compensation_failure)
14294        }
14295        Error::WorkflowCancellationRequested(_) => true,
14296        Error::NonDeterministicReplay(_) => true,
14297        _ => false,
14298    };
14299
14300    json!({
14301        "type": "fail_workflow",
14302        "message": error.to_string(),
14303        "exception_type": exception_type,
14304        "exception_class": exception_class,
14305        "non_retryable": non_retryable,
14306        "exception": {
14307            "type": exception_type,
14308            "class": exception_class,
14309            "message": error.to_string(),
14310            "properties": properties,
14311        }
14312    })
14313}
14314
14315fn workflow_error_type(error: &Error) -> &'static str {
14316    match error {
14317        Error::ActivityFailed(failure) => match failure.kind {
14318            ActivityFailureKind::Failed => "ActivityFailed",
14319            ActivityFailureKind::Cancelled => "ActivityCancelled",
14320            ActivityFailureKind::TimedOut => "ActivityTimedOut",
14321        },
14322        Error::ChildWorkflowFailed(failure) => match failure.kind {
14323            ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14324            ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14325            ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14326        },
14327        Error::ParallelFailed(_) => "ParallelFailed",
14328        Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14329        Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14330        Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14331        _ => "RustWorkflowError",
14332    }
14333}
14334
14335fn workflow_error_non_retryable(error: &Error) -> bool {
14336    match error {
14337        Error::ActivityFailed(failure) => failure.non_retryable,
14338        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14339        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14340        Error::SagaCompensationFailed(failure) => {
14341            workflow_error_non_retryable(&failure.compensation_failure)
14342        }
14343        Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14344        _ => false,
14345    }
14346}
14347
14348fn workflow_task_integrity_error(error: &Error) -> bool {
14349    matches!(
14350        error,
14351        Error::NonDeterministicReplay(_)
14352            | Error::Protocol(_)
14353            | Error::MissingWorkflowCommandIdentity
14354            | Error::WorkflowStatePoisoned
14355    )
14356}
14357
14358fn decode_signal_event_arguments(
14359    event: &HistoryEvent,
14360    fallback_codec: &str,
14361) -> Result<Vec<AvroValue>> {
14362    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14363    validate_payload_codec(codec)?;
14364    let raw = signal_history_payload(&event.payload);
14365    let decoded = match raw.filter(|value| !value.is_null()) {
14366        Some(value) => decode_wire_avro_value(value, codec)?,
14367        None => AvroValue::Array(Vec::new()),
14368    };
14369    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14370        unreachable!("normalize_avro_arguments always returns an array");
14371    };
14372    Ok(arguments)
14373}
14374
14375fn decode_update_event_arguments(
14376    event: &HistoryEvent,
14377    fallback_codec: &str,
14378) -> Result<Vec<AvroValue>> {
14379    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14380    validate_payload_codec(codec)?;
14381    let decoded = match event
14382        .payload
14383        .get("arguments")
14384        .filter(|value| !value.is_null())
14385    {
14386        Some(value) => decode_wire_avro_value(value, codec)?,
14387        None => AvroValue::Array(Vec::new()),
14388    };
14389    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14390        unreachable!("normalize_avro_arguments always returns an array");
14391    };
14392    Ok(arguments)
14393}
14394
14395fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14396    let Some(export_events) = task
14397        .history_export
14398        .as_ref()
14399        .and_then(|export| export.get("history_events"))
14400        .and_then(Value::as_array)
14401    else {
14402        return Ok(());
14403    };
14404
14405    if export_events.len() > task.history_events.len() {
14406        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14407    }
14408
14409    Ok(())
14410}
14411
14412fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14413    let Some(export) = task.history_export.as_ref() else {
14414        return Ok(());
14415    };
14416    let signals = export
14417        .get("signals")
14418        .and_then(Value::as_array)
14419        .cloned()
14420        .unwrap_or_default();
14421    let activities = export
14422        .get("activities")
14423        .and_then(Value::as_array)
14424        .cloned()
14425        .unwrap_or_default();
14426    let export_codec = export
14427        .get("payloads")
14428        .and_then(|payloads| payloads.get("codec"))
14429        .and_then(Value::as_str)
14430        .unwrap_or(&task.payload_codec)
14431        .to_string();
14432    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14433
14434    for event in &mut task.history_events {
14435        if event.event_type == "ActivityCompleted" {
14436            let sequence = event
14437                .payload
14438                .get("sequence")
14439                .or_else(|| event.payload.get("workflow_sequence"))
14440                .and_then(value_as_u64);
14441            let Some(activity) = sequence.and_then(|sequence| {
14442                activities.iter().find(|activity| {
14443                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14444                })
14445            }) else {
14446                continue;
14447            };
14448            let Some(payload) = event.payload.as_object_mut() else {
14449                continue;
14450            };
14451            if missing_payload(payload.get("result")) {
14452                if let Some(result) = activity
14453                    .get("result")
14454                    .filter(|value| !missing_payload(Some(value)))
14455                {
14456                    payload.insert("result".to_string(), result.clone());
14457                }
14458            }
14459            for field in ["payload_codec", "activity_type"] {
14460                if payload
14461                    .get(field)
14462                    .and_then(Value::as_str)
14463                    .unwrap_or_default()
14464                    .is_empty()
14465                {
14466                    if let Some(value) = activity.get(field) {
14467                        payload.insert(field.to_string(), value.clone());
14468                    }
14469                }
14470            }
14471            continue;
14472        }
14473
14474        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14475            continue;
14476        }
14477        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14478        let command_id = event
14479            .payload
14480            .get("workflow_command_id")
14481            .or_else(|| event.raw.get("workflow_command_id"))
14482            .and_then(Value::as_str);
14483        let signal_name = event
14484            .payload
14485            .get("signal_name")
14486            .and_then(Value::as_str)
14487            .unwrap_or_default()
14488            .to_string();
14489        let matched = signals
14490            .iter()
14491            .find(|signal| {
14492                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14493            })
14494            .or_else(|| {
14495                signals.iter().find(|signal| {
14496                    command_id.is_some()
14497                        && signal.get("command_id").and_then(Value::as_str) == command_id
14498                })
14499            })
14500            .or_else(|| {
14501                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14502                let signal = signals
14503                    .iter()
14504                    .filter(|signal| {
14505                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14506                    })
14507                    .nth(*offset);
14508                if signal.is_some() {
14509                    *offset += 1;
14510                }
14511                signal
14512            });
14513        let Some(signal) = matched else {
14514            continue;
14515        };
14516        let signal_codec = signal
14517            .get("payload_codec")
14518            .and_then(Value::as_str)
14519            .unwrap_or(&export_codec);
14520        let Some(payload) = event.payload.as_object_mut() else {
14521            continue;
14522        };
14523        if missing_payload(payload.get("arguments")) {
14524            if let Some(arguments) = signal
14525                .get("arguments")
14526                .filter(|value| !missing_payload(Some(value)))
14527            {
14528                let envelope = match arguments {
14529                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14530                    other => other.clone(),
14531                };
14532                payload.insert("arguments".to_string(), envelope);
14533            }
14534        }
14535        if payload
14536            .get("payload_codec")
14537            .and_then(Value::as_str)
14538            .unwrap_or_default()
14539            .is_empty()
14540        {
14541            payload.insert("payload_codec".to_string(), json!(signal_codec));
14542        }
14543    }
14544
14545    Ok(())
14546}
14547
14548fn missing_payload(value: Option<&Value>) -> bool {
14549    match value {
14550        None | Some(Value::Null) => true,
14551        Some(Value::String(value)) => value.is_empty(),
14552        Some(_) => false,
14553    }
14554}
14555
14556fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14557    let export_signals = task
14558        .history_export
14559        .as_ref()
14560        .and_then(|export| export.get("signals"))
14561        .and_then(Value::as_array)
14562        .cloned()
14563        .unwrap_or_default();
14564    let export_codec = task
14565        .history_export
14566        .as_ref()
14567        .and_then(|export| export.get("payloads"))
14568        .and_then(|payloads| payloads.get("codec"))
14569        .and_then(Value::as_str)
14570        .unwrap_or(&task.payload_codec);
14571    let mut name_offsets: HashMap<String, usize> = HashMap::new();
14572    let mut signals = Vec::new();
14573
14574    for event in &task.history_events {
14575        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14576            continue;
14577        }
14578
14579        let name = event
14580            .payload
14581            .get("signal_name")
14582            .and_then(Value::as_str)
14583            .unwrap_or_default();
14584        if name.is_empty() {
14585            continue;
14586        }
14587        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14588        let command_id = event
14589            .payload
14590            .get("workflow_command_id")
14591            .or_else(|| event.raw.get("workflow_command_id"))
14592            .and_then(Value::as_str);
14593        let matched_export = export_signals
14594            .iter()
14595            .find(|candidate| {
14596                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14597            })
14598            .or_else(|| {
14599                export_signals.iter().find(|candidate| {
14600                    command_id.is_some()
14601                        && candidate.get("command_id").and_then(Value::as_str) == command_id
14602                })
14603            })
14604            .or_else(|| {
14605                let offset = name_offsets.entry(name.to_string()).or_default();
14606                let candidate = export_signals
14607                    .iter()
14608                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14609                    .nth(*offset);
14610                if candidate.is_some() {
14611                    *offset += 1;
14612                }
14613                candidate
14614            });
14615        let codec = event
14616            .payload
14617            .get("payload_codec")
14618            .and_then(Value::as_str)
14619            .or_else(|| {
14620                matched_export
14621                    .and_then(|signal| signal.get("payload_codec"))
14622                    .and_then(Value::as_str)
14623            })
14624            .unwrap_or(export_codec);
14625        let raw_arguments = signal_history_payload(&event.payload)
14626            .filter(|value| !value.is_null())
14627            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14628        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14629        let workflow_sequence = event
14630            .payload
14631            .get("workflow_sequence")
14632            .and_then(value_as_u64)
14633            .or_else(|| {
14634                matched_export
14635                    .and_then(|signal| signal.get("workflow_sequence"))
14636                    .and_then(value_as_u64)
14637            });
14638
14639        signals.push(QuerySignal {
14640            id: signal_id.map(str::to_string).or_else(|| {
14641                matched_export
14642                    .and_then(|signal| signal.get("id"))
14643                    .and_then(Value::as_str)
14644                    .map(str::to_string)
14645            }),
14646            name: name.to_string(),
14647            arguments,
14648            avro_arguments,
14649            workflow_sequence,
14650        });
14651    }
14652
14653    if signals.is_empty() {
14654        for signal in export_signals {
14655            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14656                continue;
14657            }
14658            let Some(name) = signal.get("name").and_then(Value::as_str) else {
14659                continue;
14660            };
14661            let codec = signal
14662                .get("payload_codec")
14663                .and_then(Value::as_str)
14664                .unwrap_or(export_codec);
14665            let (arguments, avro_arguments) =
14666                decode_query_signal_arguments(signal.get("arguments"), codec)?;
14667            signals.push(QuerySignal {
14668                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14669                name: name.to_string(),
14670                arguments,
14671                avro_arguments,
14672                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14673            });
14674        }
14675        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14676    }
14677
14678    Ok(signals)
14679}
14680
14681fn decode_query_signal_arguments(
14682    raw: Option<&Value>,
14683    codec: &str,
14684) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14685    validate_payload_codec(codec)?;
14686    let decoded = match raw.filter(|value| !value.is_null()) {
14687        Some(value) => decode_wire_avro_value(value, codec)?,
14688        None => AvroValue::Array(Vec::new()),
14689    };
14690    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14691        unreachable!("normalize_avro_arguments always returns an array");
14692    };
14693    let arguments = avro_arguments
14694        .iter()
14695        .cloned()
14696        .map(AvroValue::into_json)
14697        .collect::<Result<Vec<_>>>()?;
14698    Ok((arguments, avro_arguments))
14699}
14700
14701fn value_as_u64(value: &Value) -> Option<u64> {
14702    value
14703        .as_u64()
14704        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14705}
14706
14707#[cfg(test)]
14708mod tests {
14709    use super::*;
14710    use std::{
14711        fs,
14712        io::{Read, Write},
14713        net::{SocketAddr, TcpListener, TcpStream},
14714        process::Command as ProcessCommand,
14715        sync::atomic::AtomicUsize,
14716        thread,
14717    };
14718
14719    #[derive(Clone, Copy, Debug)]
14720    enum InvalidTaskPayloadCodec {
14721        Missing,
14722        Null,
14723        NonString,
14724    }
14725
14726    impl InvalidTaskPayloadCodec {
14727        fn label(self) -> &'static str {
14728            match self {
14729                Self::Missing => "missing",
14730                Self::Null => "null",
14731                Self::NonString => "non-string",
14732            }
14733        }
14734
14735        fn apply(self, task: &mut Value) {
14736            let task = task.as_object_mut().expect("task fixture object");
14737            match self {
14738                Self::Missing => {
14739                    task.remove("payload_codec");
14740                }
14741                Self::Null => {
14742                    task.insert("payload_codec".to_string(), Value::Null);
14743                }
14744                Self::NonString => {
14745                    task.insert("payload_codec".to_string(), json!(42));
14746                }
14747            }
14748        }
14749    }
14750
14751    fn fixture_envelope(value: Value) -> Value {
14752        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14753    }
14754
14755    fn fixture_blob(value: Value) -> String {
14756        encode_payload(&value, DEFAULT_CODEC)
14757            .expect("encode Avro test fixture")
14758            .blob
14759    }
14760
14761    #[test]
14762    fn client_builder_rejects_the_sdk_owned_api_suffix() {
14763        for base_url in [
14764            "http://127.0.0.1:8080/api",
14765            "http://localhost:8080/api/",
14766            "https://runtime.example.test/namespaces/orders/api",
14767        ] {
14768            let error = Client::builder(base_url)
14769                .build()
14770                .expect_err("SDK-owned /api suffix must be rejected during build");
14771
14772            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14773            assert!(
14774                error.to_string().contains("SDK appends /api automatically"),
14775                "the validation error must explain how to fix the endpoint"
14776            );
14777        }
14778    }
14779
14780    #[test]
14781    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14782        for (base_url, expected) in [
14783            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14784            (
14785                "http://localhost:8080/durable-workflow/",
14786                "http://localhost:8080/durable-workflow",
14787            ),
14788            (
14789                "https://runtime.example.test/namespaces/orders",
14790                "https://runtime.example.test/namespaces/orders",
14791            ),
14792            (
14793                "https://runtime.example.test/gateway/api/namespaces/orders",
14794                "https://runtime.example.test/gateway/api/namespaces/orders",
14795            ),
14796            (
14797                "https://api.example.test/runtime/orders/",
14798                "https://api.example.test/runtime/orders",
14799            ),
14800        ] {
14801            let client = Client::builder(base_url)
14802                .build()
14803                .expect("Server and Cloud runtime base URL must remain valid");
14804
14805            assert_eq!(client.base_url, expected);
14806        }
14807    }
14808
14809    #[test]
14810    fn workflow_completion_uses_the_additive_command_protocol_floor() {
14811        assert_eq!(
14812            workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14813            WORKER_PROTOCOL_VERSION
14814        );
14815        assert_eq!(
14816            workflow_completion_protocol_version(&[json!({
14817                "type": "upsert_search_attributes",
14818                "attributes": {"OrderStatus": "waiting"},
14819            })]),
14820            SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14821        );
14822        assert_eq!(
14823            workflow_completion_protocol_version(&[json!({
14824                "type": "upsert_search_attributes",
14825                "attributes": {"OrderStatus": "waiting"},
14826                "attribute_types": {"OrderStatus": "keyword"},
14827            })]),
14828            TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14829        );
14830        assert_eq!(
14831            workflow_completion_protocol_version(&[
14832                json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14833                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14834            ]),
14835            MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14836        );
14837        assert_eq!(
14838            workflow_completion_protocol_version(&[
14839                json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14840                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14841            ]),
14842            CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14843        );
14844        assert_eq!(
14845            workflow_completion_protocol_version(&[json!({
14846                "type": "open_condition_wait",
14847                "condition_wait_occurrence_id": "rust:condition-wait:0",
14848                "condition_key": "ready",
14849            })]),
14850            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14851        );
14852        assert_eq!(
14853            workflow_completion_protocol_version_with_message_streams(
14854                &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14855                true,
14856            ),
14857            MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14858        );
14859        assert_eq!(
14860            workflow_completion_protocol_version_with_message_streams(
14861                &[json!({
14862                    "type": "open_condition_wait",
14863                    "condition_wait_occurrence_id": "rust:condition-wait:0",
14864                    "condition_key": "ready",
14865                })],
14866                true,
14867            ),
14868            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14869        );
14870    }
14871
14872    #[test]
14873    fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14874        let manifest = portable_worker_affinity_capability_manifest();
14875
14876        for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14877            assert_eq!(manifest[capability]["supported"], json!(false));
14878            assert_eq!(
14879                manifest[capability]["minimum_protocol_version"],
14880                json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14881            );
14882            assert!(manifest[capability]["reason"]
14883                .as_str()
14884                .is_some_and(|reason| !reason.is_empty()));
14885        }
14886    }
14887
14888    fn typed_fidelity_probe() -> AvroValue {
14889        AvroValue::Map(BTreeMap::from([
14890            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
14891            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
14892            (
14893                "numeric".to_string(),
14894                AvroValue::Map(BTreeMap::from([
14895                    ("0".to_string(), AvroValue::String("zero".to_string())),
14896                    ("1".to_string(), AvroValue::String("one".to_string())),
14897                ])),
14898            ),
14899            (
14900                "nested".to_string(),
14901                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
14902                    "enabled".to_string(),
14903                    AvroValue::Boolean(true),
14904                )]))]),
14905            ),
14906            (
14907                "projection_collisions".to_string(),
14908                AvroValue::Array(projection_collision_probe()),
14909            ),
14910        ]))
14911    }
14912
14913    fn projection_collision_probe() -> Vec<AvroValue> {
14914        vec![
14915            AvroValue::Map(BTreeMap::from([
14916                ("$type".to_string(), AvroValue::String("bytes".to_string())),
14917                (
14918                    "base64".to_string(),
14919                    AvroValue::String("ordinary user text".to_string()),
14920                ),
14921            ])),
14922            AvroValue::Map(BTreeMap::from([
14923                ("$type".to_string(), AvroValue::String("map".to_string())),
14924                (
14925                    "entries".to_string(),
14926                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
14927                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
14928                        (
14929                            "value".to_string(),
14930                            AvroValue::String("user map".to_string()),
14931                        ),
14932                    ]))]),
14933                ),
14934            ])),
14935        ]
14936    }
14937
14938    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14939    struct TypedContract {
14940        nested: TypedNested,
14941        mode: TypedMode,
14942        optional: Option<String>,
14943        absent: Option<String>,
14944        items: Vec<i64>,
14945        labels: BTreeMap<String, String>,
14946        bytes: serde_bytes::ByteBuf,
14947        signed: i64,
14948        finite: f64,
14949    }
14950
14951    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14952    struct TypedNested {
14953        enabled: bool,
14954    }
14955
14956    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14957    enum TypedMode {
14958        Detailed { label: String },
14959    }
14960
14961    fn typed_contract() -> TypedContract {
14962        TypedContract {
14963            nested: TypedNested { enabled: true },
14964            mode: TypedMode::Detailed {
14965                label: "compiler-checked".to_string(),
14966            },
14967            optional: Some("present".to_string()),
14968            absent: None,
14969            items: vec![i64::MIN, 0, i64::MAX],
14970            labels: BTreeMap::from([
14971                ("language".to_string(), "rust".to_string()),
14972                ("wire".to_string(), "avro".to_string()),
14973            ]),
14974            bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
14975            signed: -9_223_372_036_854_775_000,
14976            finite: 12.5,
14977        }
14978    }
14979
14980    #[derive(Clone, Debug, Default, PartialEq)]
14981    struct ReplayCounterState {
14982        loaded: Option<String>,
14983        count: i64,
14984        finished: bool,
14985    }
14986
14987    fn replay_counter_worker() -> Worker {
14988        let client = Client::new("http://127.0.0.1:8080").expect("client");
14989        let mut worker = Worker::new(client, "rust-workers");
14990        worker.register_replayed_workflow(
14991            "replay-counter",
14992            ReplayCounterState::default,
14993            |ctx, _input, state| async move {
14994                let loaded = ctx.activity("load-counter", json!([])).await?;
14995                state.update(|current| {
14996                    current.loaded = loaded.as_str().map(str::to_string);
14997                })?;
14998                for _ in 0..2 {
14999                    let signal = ctx.wait_signal("increment").await?;
15000                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15001                    state.update(|current| current.count += amount)?;
15002                }
15003                state.update(|current| current.finished = true)?;
15004                state.read(|current| Ok(json!(current.count)))?
15005            },
15006        );
15007        worker.register_replayed_query::<ReplayCounterState, _, _>(
15008            "replay-counter",
15009            "current",
15010            |_ctx, state, _args| async move {
15011                Ok(json!({
15012                    "loaded": state.loaded,
15013                    "count": state.count,
15014                    "finished": state.finished,
15015                }))
15016            },
15017        );
15018        worker.register_replayed_query::<ReplayCounterState, _, _>(
15019            "replay-counter",
15020            "detached-mutation",
15021            |_ctx, state, _args| async move {
15022                let mut detached = (*state).clone();
15023                detached.count = 999;
15024                Ok(json!(detached.count))
15025            },
15026        );
15027        worker.register_replayed_query::<ReplayCounterState, _, _>(
15028            "replay-counter",
15029            "failed-mutation",
15030            |_ctx, state, _args| async move {
15031                let mut detached = (*state).clone();
15032                detached.count = 999;
15033                Err(Error::WorkerLoop("query refused".to_string()))
15034            },
15035        );
15036        worker
15037    }
15038
15039    fn replay_counter_query(
15040        query_name: &str,
15041        history_events: Value,
15042        run_status: &str,
15043    ) -> QueryTask {
15044        let arguments = fixture_envelope(json!([]));
15045        serde_json::from_value(json!({
15046            "query_task_id": format!("query-{query_name}"),
15047            "workflow_type": "replay-counter",
15048            "query_name": query_name,
15049            "payload_codec": DEFAULT_CODEC,
15050            "workflow_arguments": arguments.clone(),
15051            "query_arguments": arguments,
15052            "history_events": history_events,
15053            "run_status": run_status,
15054        }))
15055        .expect("query task")
15056    }
15057
15058    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15059        workflow_context_with_codec(history, DEFAULT_CODEC)
15060    }
15061
15062    fn workflow_context_with_codec(
15063        history: Vec<HistoryEvent>,
15064        payload_codec: &str,
15065    ) -> WorkflowContext {
15066        WorkflowContext {
15067            state: Arc::new(Mutex::new(
15068                WorkflowState::new_with_identity(
15069                    history,
15070                    None,
15071                    None,
15072                    "rust-workers".to_string(),
15073                    payload_codec.to_string(),
15074                    None,
15075                )
15076                .expect("valid workflow history"),
15077            )),
15078        }
15079    }
15080
15081    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15082        HistoryEvent {
15083            event_type: event_type.to_string(),
15084            payload,
15085            raw: HashMap::new(),
15086        }
15087    }
15088
15089    fn parallel_path_entry(
15090        kind: &str,
15091        base: u64,
15092        size: usize,
15093        index: usize,
15094    ) -> ParallelGroupMetadata {
15095        parallel_group_entry(base, size, index, kind)
15096    }
15097
15098    fn parallel_history_event(
15099        event_type: &str,
15100        sequence: u64,
15101        identity_field: &str,
15102        identity: &str,
15103        path: Vec<ParallelGroupMetadata>,
15104        result: Option<Value>,
15105    ) -> HistoryEvent {
15106        let mut payload = serde_json::Map::from_iter([
15107            ("sequence".to_string(), json!(sequence)),
15108            (identity_field.to_string(), json!(identity)),
15109        ]);
15110        let inner = path.last().expect("parallel history path");
15111        apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15112        payload.insert("parallel_group_path".to_string(), json!(path));
15113        if let Some(result) = result {
15114            let field = if event_type == "ChildRunCompleted" {
15115                "result"
15116            } else {
15117                "result"
15118            };
15119            payload.insert(field.to_string(), fixture_envelope(result));
15120            payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15121        }
15122        history_event(event_type, Value::Object(payload))
15123    }
15124
15125    fn nested_parallel_operations() -> Vec<ParallelOperation> {
15126        vec![
15127            ParallelOperation::activity("first", json!([])),
15128            ParallelOperation::group(vec![
15129                ParallelOperation::child_workflow(
15130                    "second",
15131                    ChildWorkflowOptions::new("child-workers"),
15132                    json!([]),
15133                ),
15134                ParallelOperation::activity("third", json!([])),
15135            ]),
15136        ]
15137    }
15138
15139    fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15140        let outer = [
15141            parallel_path_entry("mixed", 1, 3, 0),
15142            parallel_path_entry("mixed", 1, 3, 1),
15143            parallel_path_entry("mixed", 1, 3, 2),
15144        ];
15145        [
15146            vec![outer[0].clone()],
15147            vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15148            vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15149        ]
15150    }
15151
15152    #[test]
15153    fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15154        let ctx = workflow_context(Vec::new());
15155        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15156        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15157
15158        assert!(matches!(
15159            call.as_mut().poll(&mut task_context),
15160            Poll::Pending
15161        ));
15162        let commands = ctx.take_commands().expect("parallel commands");
15163        assert_eq!(
15164            commands
15165                .iter()
15166                .map(|command| command["type"].as_str().unwrap_or_default())
15167                .collect::<Vec<_>>(),
15168            [
15169                "schedule_activity",
15170                "start_child_workflow",
15171                "schedule_activity"
15172            ]
15173        );
15174        let paths = nested_parallel_paths();
15175        for (command, path) in commands.iter().zip(paths) {
15176            assert_eq!(command["parallel_group_path"], json!(path));
15177            assert_eq!(
15178                command["parallel_group_id"],
15179                json!(path.last().expect("inner group").parallel_group_id)
15180            );
15181        }
15182    }
15183
15184    fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15185        let paths = nested_parallel_paths();
15186        let third = parallel_history_event(
15187            "ActivityCompleted",
15188            3,
15189            "activity_type",
15190            "third",
15191            paths[2].clone(),
15192            Some(json!("three")),
15193        );
15194        vec![
15195            parallel_history_event(
15196                "ActivityCompleted",
15197                1,
15198                "activity_type",
15199                "first",
15200                paths[0].clone(),
15201                Some(json!("one")),
15202            ),
15203            parallel_history_event(
15204                "ChildWorkflowScheduled",
15205                2,
15206                "child_workflow_type",
15207                "second",
15208                paths[1].clone(),
15209                None,
15210            ),
15211            parallel_history_event(
15212                "ChildRunCompleted",
15213                2,
15214                "child_workflow_type",
15215                "second",
15216                paths[1].clone(),
15217                Some(json!("two")),
15218            ),
15219            third.clone(),
15220            third,
15221        ]
15222    }
15223
15224    #[test]
15225    fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15226        for _restart_or_completed_replay in 0..2 {
15227            let ctx = workflow_context(completed_nested_parallel_history());
15228            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15229            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15230            let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15231                panic!("completed nested parallel history must replay");
15232            };
15233            assert_eq!(
15234                results,
15235                vec![
15236                    ParallelResult::Activity(json!("one")),
15237                    ParallelResult::Group(vec![
15238                        ParallelResult::ChildWorkflow(ChildWorkflowResult {
15239                            parent: WorkflowIdentity {
15240                                workflow_id: None,
15241                                run_id: None,
15242                            },
15243                            child: WorkflowIdentity {
15244                                workflow_id: None,
15245                                run_id: None,
15246                            },
15247                            child_workflow_type: Some("second".to_string()),
15248                            result: json!("two"),
15249                        }),
15250                        ParallelResult::Activity(json!("three")),
15251                    ]),
15252                ]
15253            );
15254            assert!(ctx.take_commands().expect("commands").is_empty());
15255            ctx.ensure_history_consumed().expect("history consumed");
15256        }
15257    }
15258
15259    #[test]
15260    fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15261        let paths = nested_parallel_paths();
15262        let history = vec![
15263            parallel_history_event(
15264                "ActivityCompleted",
15265                1,
15266                "activity_type",
15267                "first",
15268                paths[0].clone(),
15269                Some(json!("one")),
15270            ),
15271            parallel_history_event(
15272                "ChildWorkflowScheduled",
15273                2,
15274                "child_workflow_type",
15275                "second",
15276                paths[1].clone(),
15277                None,
15278            ),
15279            parallel_history_event(
15280                "ChildRunFailed",
15281                2,
15282                "child_workflow_type",
15283                "second",
15284                paths[1].clone(),
15285                None,
15286            ),
15287            parallel_history_event(
15288                "ActivityCompleted",
15289                3,
15290                "activity_type",
15291                "third",
15292                paths[2].clone(),
15293                Some(json!("late")),
15294            ),
15295        ];
15296        let ctx = workflow_context(history);
15297        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15298        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15299        let outcome = call.as_mut().poll(&mut task_context);
15300        let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15301            panic!("one failed child must return a typed partial failure: {outcome:?}");
15302        };
15303        assert_eq!(failure.member_path, [1, 0]);
15304        assert_eq!(failure.group_id, "parallel-calls:1:3");
15305        assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15306        assert_eq!(
15307            failure
15308                .completed
15309                .iter()
15310                .map(|completion| completion.member_path.clone())
15311                .collect::<Vec<_>>(),
15312            [vec![0], vec![1, 1]]
15313        );
15314    }
15315
15316    #[test]
15317    fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15318        let paths = nested_parallel_paths();
15319        let history = vec![
15320            parallel_history_event(
15321                "ActivityScheduled",
15322                1,
15323                "activity_type",
15324                "first",
15325                paths[0].clone(),
15326                None,
15327            ),
15328            parallel_history_event(
15329                "ChildWorkflowScheduled",
15330                2,
15331                "child_workflow_type",
15332                "second",
15333                paths[1].clone(),
15334                None,
15335            ),
15336            parallel_history_event(
15337                "ActivityScheduled",
15338                3,
15339                "activity_type",
15340                "third",
15341                paths[2].clone(),
15342                None,
15343            ),
15344        ];
15345        for _restart in 0..2 {
15346            let ctx = workflow_context(history.clone());
15347            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15348            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15349            let outcome = call.as_mut().poll(&mut task_context);
15350            assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15351            assert!(ctx.take_commands().expect("commands").is_empty());
15352        }
15353    }
15354
15355    fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15356        vec![selection_group_entry(
15357            1,
15358            2,
15359            index,
15360            "activity",
15361            &SelectionMemberMetadata {
15362                key: SelectionKey::Name(key.to_string()),
15363                index,
15364                base_sequence: index as u64 + 1,
15365                size: 1,
15366                kind: "activity".to_string(),
15367            },
15368        )]
15369    }
15370
15371    fn selection_activity_event(
15372        event_type: &str,
15373        index: usize,
15374        key: &str,
15375        result: Option<Value>,
15376    ) -> HistoryEvent {
15377        let sequence = index as u64 + 1;
15378        let mut event = parallel_history_event(
15379            event_type,
15380            sequence,
15381            "activity_type",
15382            &format!("{key}-activity"),
15383            selection_path(index, key),
15384            result,
15385        );
15386        event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15387        event.raw.insert(
15388            "id".to_string(),
15389            json!(if event_type == "ActivityCompleted" {
15390                format!("event-{key}")
15391            } else {
15392                format!("{event_type}-{key}")
15393            }),
15394        );
15395        event
15396    }
15397
15398    fn selection_winner_marker() -> HistoryEvent {
15399        history_event(
15400            "SelectionResolved",
15401            json!({
15402                "selection_group_id": "select-calls:1:2",
15403                "selection_group_base_sequence": 1,
15404                "selection_group_size": 2,
15405                "member_key": "fast",
15406                "member_index": 1,
15407                "member_base_sequence": 2,
15408                "member_size": 1,
15409                "operation_kind": "activity",
15410                "operation_identity": "activity-fast",
15411                "outcome": "completed",
15412                "resolution_event_id": "event-fast",
15413                "resolution_event_type": "ActivityCompleted",
15414            }),
15415        )
15416    }
15417
15418    fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15419        ctx.select_keyed(vec![
15420            (
15421                "slow",
15422                ParallelOperation::activity_with_options(
15423                    "slow-activity",
15424                    ActivityOptions::new().task_queue("default"),
15425                    json!([]),
15426                ),
15427            ),
15428            (
15429                "fast",
15430                ParallelOperation::activity_with_options(
15431                    "fast-activity",
15432                    ActivityOptions::new().task_queue("default"),
15433                    json!([]),
15434                ),
15435            ),
15436        ])
15437    }
15438
15439    fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15440        let ctx = workflow_context(history);
15441        let mut call = Box::pin(keyed_activity_selection(&ctx));
15442        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15443        let selected = match call.as_mut().poll(&mut task_context) {
15444            Poll::Ready(Ok(selected)) => selected,
15445            Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15446            Poll::Pending => panic!("persisted selection winner must replay without pending"),
15447        };
15448        assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15449        assert_eq!(
15450            selected.value,
15451            Some(ParallelResult::Activity(json!("winner-value")))
15452        );
15453        let slow = selected
15454            .handle(&SelectionKey::Name("slow".to_string()))
15455            .expect("slow handle")
15456            .clone();
15457        let mut await_slow = Box::pin(slow.await_result());
15458        assert!(matches!(
15459            await_slow.as_mut().poll(&mut task_context),
15460            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15461        ));
15462        assert!(ctx.take_commands().expect("commands").is_empty());
15463    }
15464
15465    const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15466
15467    fn canonical_selection_history() -> Vec<HistoryEvent> {
15468        const FIXTURE: &[u8] =
15469            include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15470        assert_eq!(
15471            format!("{:x}", Sha256::digest(FIXTURE)),
15472            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15473        );
15474        let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15475
15476        serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15477    }
15478
15479    #[test]
15480    fn selection_fresh_process_entrypoint() {
15481        let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15482            return;
15483        };
15484        let persisted = fs::read(path).expect("persisted selection history");
15485        assert_eq!(
15486            format!("{:x}", Sha256::digest(&persisted)),
15487            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15488        );
15489        let fixture: Value =
15490            serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15491        let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15492            .expect("valid persisted selection history");
15493
15494        assert_persisted_selection_replay(history);
15495    }
15496
15497    #[test]
15498    fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15499        let ctx = workflow_context(Vec::new());
15500        let mut call = Box::pin(keyed_activity_selection(&ctx));
15501        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15502
15503        assert!(matches!(
15504            call.as_mut().poll(&mut task_context),
15505            Poll::Pending
15506        ));
15507        let commands = ctx.take_commands().expect("selection commands");
15508        assert_eq!(commands.len(), 2);
15509        assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15510        assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15511        assert!(commands.iter().all(|command| {
15512            command["parallel_group_id"] == json!("select-calls:1:2")
15513                && command["parallel_group_mode"] == json!("select")
15514        }));
15515    }
15516
15517    #[test]
15518    fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15519        let ctx = workflow_context(Vec::new());
15520        let mut invalid = Box::pin(ctx.select_keyed(vec![(
15521            "",
15522            ParallelOperation::activity("invalid", json!([])),
15523        )]));
15524        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15525        assert!(matches!(
15526            invalid.as_mut().poll(&mut task_context),
15527            Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15528                reason: "selection_key_invalid",
15529                ..
15530            })))
15531        ));
15532
15533        for invalid_key in [json!(""), json!(-1)] {
15534            let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15535            event.payload["selection_member_key"] = invalid_key.clone();
15536            event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15537            assert!(matches!(
15538                WorkflowState::new_with_identity(
15539                    vec![event],
15540                    None,
15541                    None,
15542                    "rust-workers".to_string(),
15543                    DEFAULT_CODEC.to_string(),
15544                    None,
15545                ),
15546                Err(Error::NonDeterministicReplay(_))
15547            ));
15548        }
15549    }
15550
15551    #[test]
15552    fn selection_preserves_valid_named_and_numeric_keys() {
15553        let ctx = workflow_context(Vec::new());
15554        let mut selection = Box::pin(ctx.select_keyed(vec![
15555            (
15556                SelectionKey::Index(0),
15557                ParallelOperation::activity("numeric", json!([])),
15558            ),
15559            (
15560                SelectionKey::Name("named".to_string()),
15561                ParallelOperation::timer(Duration::from_secs(1)),
15562            ),
15563        ]));
15564        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15565
15566        assert!(matches!(
15567            selection.as_mut().poll(&mut task_context),
15568            Poll::Pending
15569        ));
15570        let commands = ctx.take_commands().expect("selection commands");
15571        assert_eq!(commands[0]["selection_member_key"], json!(0));
15572        assert_eq!(commands[1]["selection_member_key"], json!("named"));
15573    }
15574
15575    #[test]
15576    fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15577        let history = canonical_selection_history();
15578        assert_persisted_selection_replay(history.clone());
15579
15580        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15581            .join("tests/fixtures/durable_selection_runtime_history.json");
15582        let output =
15583            ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15584                .args([
15585                    "--exact",
15586                    "tests::selection_fresh_process_entrypoint",
15587                    "--nocapture",
15588                ])
15589                .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15590                .output()
15591                .expect("run fresh selection replay process");
15592
15593        assert!(
15594            output.status.success(),
15595            "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15596            String::from_utf8_lossy(&output.stdout),
15597            String::from_utf8_lossy(&output.stderr),
15598        );
15599    }
15600
15601    #[test]
15602    fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15603        let mut history = canonical_selection_history();
15604        history.retain(|event| event.event_type != "SelectionResolved");
15605        let ctx = workflow_context(history);
15606        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15607        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15608
15609        assert!(matches!(
15610            selection.as_mut().poll(&mut task_context),
15611            Poll::Pending
15612        ));
15613        assert!(ctx.take_commands().expect("commands").is_empty());
15614        assert!(
15615            ctx.matched_recorded_pending()
15616                .expect("selection pending state"),
15617            "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15618        );
15619    }
15620
15621    #[test]
15622    fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15623        for (terminal_event, predicate_satisfied, timeout_seconds) in [
15624            ("ConditionWaitSatisfied", true, None),
15625            ("ConditionWaitTimedOut", false, Some(0)),
15626        ] {
15627            let member = SelectionMemberMetadata {
15628                key: SelectionKey::Name("condition".to_string()),
15629                index: 0,
15630                base_sequence: 1,
15631                size: 1,
15632                kind: "condition".to_string(),
15633            };
15634            let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15635            let mut payload = json!({
15636                "sequence": 1,
15637                "condition_wait_id": "condition-1",
15638                "condition_wait_occurrence_id": "rust:condition-wait:0",
15639                "condition_key": "ready",
15640                "condition_definition_fingerprint": "sha256:ready-v1",
15641                "parallel_group_path": path,
15642            });
15643            payload
15644                .as_object_mut()
15645                .expect("condition history payload")
15646                .extend(
15647                    serde_json::to_value(&path[0])
15648                        .expect("condition selection metadata")
15649                        .as_object()
15650                        .expect("condition selection metadata object")
15651                        .clone(),
15652                );
15653            if let Some(timeout_seconds) = timeout_seconds {
15654                payload["timeout_seconds"] = json!(timeout_seconds);
15655            }
15656            let history = vec![
15657                history_event("ConditionWaitOpened", payload.clone()),
15658                history_event(terminal_event, payload),
15659            ];
15660            let ctx = workflow_context(history);
15661            let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15662            if timeout_seconds.is_some() {
15663                options = options.timeout(Duration::ZERO);
15664            }
15665            let mut selection = Box::pin(ctx.select_keyed(vec![(
15666                "condition",
15667                ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15668            )]));
15669            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15670
15671            assert!(matches!(
15672                selection.as_mut().poll(&mut task_context),
15673                Poll::Pending
15674            ));
15675            assert!(ctx.take_commands().expect("commands").is_empty());
15676            assert!(
15677                ctx.matched_recorded_pending()
15678                    .expect("condition selection pending state"),
15679                "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15680            );
15681        }
15682    }
15683
15684    #[test]
15685    fn selection_immediate_condition_members_open_a_durable_wait() {
15686        for predicate_satisfied in [true, false] {
15687            let ctx = workflow_context(Vec::new());
15688            let mut selection = Box::pin(ctx.select_keyed(vec![(
15689                "condition",
15690                ParallelOperation::condition(
15691                    ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15692                    move || Ok(predicate_satisfied),
15693                ),
15694            )]));
15695            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15696
15697            assert!(matches!(
15698                selection.as_mut().poll(&mut task_context),
15699                Poll::Pending
15700            ));
15701            let commands = ctx.take_commands().expect("condition selection command");
15702            assert_eq!(commands.len(), 1);
15703            assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15704            assert_eq!(commands[0]["timeout_seconds"], json!(0));
15705            assert_eq!(
15706                commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15707                json!("select")
15708            );
15709        }
15710    }
15711
15712    #[test]
15713    fn selection_loser_cancellation_is_explicit_and_idempotent() {
15714        let history = vec![
15715            selection_activity_event("ActivityScheduled", 0, "slow", None),
15716            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15717            selection_winner_marker(),
15718        ];
15719        let ctx = workflow_context(history.clone());
15720        let mut call = Box::pin(keyed_activity_selection(&ctx));
15721        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15722        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15723            panic!("winner must replay");
15724        };
15725        let slow = selected
15726            .handle(&SelectionKey::Name("slow".to_string()))
15727            .expect("slow handle")
15728            .clone();
15729        let mut cancel = Box::pin(slow.cancel());
15730        assert!(matches!(
15731            cancel.as_mut().poll(&mut task_context),
15732            Poll::Pending
15733        ));
15734        assert!(matches!(
15735            cancel.as_mut().poll(&mut task_context),
15736            Poll::Pending
15737        ));
15738        let commands = ctx.take_commands().expect("cancel command");
15739        assert_eq!(commands.len(), 1);
15740        assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15741        assert_eq!(commands[0]["member_key"], json!("slow"));
15742
15743        let mut cancelled_history = history;
15744        cancelled_history.push(history_event(
15745            "SelectionOperationCancelled",
15746            json!({
15747                "selection_group_id": "select-calls:1:2",
15748                "member_key": "slow",
15749                "member_index": 0,
15750                "member_base_sequence": 1,
15751                "member_size": 1,
15752                "operation_kind": "activity",
15753                "operation_identity": "activity-slow",
15754                "cancelled_at": "2026-08-27T00:00:00Z",
15755            }),
15756        ));
15757        let replayed = workflow_context(cancelled_history);
15758        let mut call = Box::pin(keyed_activity_selection(&replayed));
15759        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15760            panic!("winner must replay after cancellation");
15761        };
15762        let slow = selected
15763            .handle(&SelectionKey::Name("slow".to_string()))
15764            .expect("slow handle")
15765            .clone();
15766        let mut cancel = Box::pin(slow.cancel());
15767        assert!(matches!(
15768            cancel.as_mut().poll(&mut task_context),
15769            Poll::Ready(Ok(()))
15770        ));
15771        assert!(replayed.take_commands().expect("commands").is_empty());
15772    }
15773
15774    #[test]
15775    fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15776        let base_history = vec![
15777            selection_activity_event("ActivityScheduled", 0, "slow", None),
15778            selection_activity_event("ActivityScheduled", 1, "fast", None),
15779            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15780            selection_winner_marker(),
15781        ];
15782        for (field, corrupt) in [
15783            ("member_key", json!("fast")),
15784            ("member_index", json!(1)),
15785            ("member_base_sequence", json!(3)),
15786            ("member_size", json!(2)),
15787            ("operation_kind", json!("timer")),
15788            ("operation_identity", json!("forged")),
15789        ] {
15790            let mut cancellation = json!({
15791                "selection_group_id": "select-calls:1:2",
15792                "member_key": "slow",
15793                "member_index": 0,
15794                "member_base_sequence": 1,
15795                "member_size": 1,
15796                "operation_kind": "activity",
15797                "operation_identity": "activity-slow",
15798            });
15799            cancellation[field] = corrupt;
15800            let mut history = base_history.clone();
15801            history.push(history_event("SelectionOperationCancelled", cancellation));
15802            let ctx = workflow_context(history);
15803            let mut selection = Box::pin(keyed_activity_selection(&ctx));
15804            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15805
15806            assert!(matches!(
15807                selection.as_mut().poll(&mut task_context),
15808                Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15809            ));
15810        }
15811    }
15812
15813    #[test]
15814    fn selection_child_identity_prefers_the_durable_run_id() {
15815        let ctx = workflow_context(vec![history_event(
15816            "ChildWorkflowScheduled",
15817            json!({
15818                "sequence": 1,
15819                "child_workflow_type": "child",
15820                "child_workflow_instance_id": "child-instance",
15821                "child_workflow_run_id": "child-run",
15822            }),
15823        )]);
15824        let state = ctx.state.lock().expect("workflow state");
15825
15826        assert_eq!(
15827            selection_operation_identity(&state, "child", 1, 1),
15828            "child-run"
15829        );
15830    }
15831
15832    #[test]
15833    fn selection_activity_identity_requires_canonical_execution_id() {
15834        let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15835        let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15836        let mut fast_completed =
15837            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15838        for event in [&mut fast_open, &mut fast_completed] {
15839            event
15840                .payload
15841                .as_object_mut()
15842                .expect("activity payload")
15843                .remove("activity_execution_id");
15844            event.payload["activity_id"] = json!("forged-activity-id");
15845        }
15846        let mut marker = selection_winner_marker();
15847        marker.payload["operation_identity"] = json!("forged-activity-id");
15848        let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15849        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15850        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15851
15852        assert!(matches!(
15853            selection.as_mut().poll(&mut task_context),
15854            Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15855        ));
15856    }
15857
15858    #[test]
15859    fn selection_completion_before_cancellation_remains_awaitable() {
15860        let history = vec![
15861            selection_activity_event("ActivityScheduled", 0, "slow", None),
15862            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15863            selection_winner_marker(),
15864            selection_activity_event(
15865                "ActivityCompleted",
15866                0,
15867                "slow",
15868                Some(json!("completed-first")),
15869            ),
15870        ];
15871        let ctx = workflow_context(history);
15872        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15873        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15874        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15875            panic!("winner must replay");
15876        };
15877        let slow = selected
15878            .handle(&SelectionKey::Name("slow".to_string()))
15879            .expect("slow handle")
15880            .clone();
15881        let mut cancel = Box::pin(slow.cancel());
15882        assert!(matches!(
15883            cancel.as_mut().poll(&mut task_context),
15884            Poll::Ready(Ok(()))
15885        ));
15886        let mut await_slow = Box::pin(slow.await_result());
15887        assert!(matches!(
15888            await_slow.as_mut().poll(&mut task_context),
15889            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
15890        ));
15891        let commands = ctx.take_commands().expect("commands");
15892        assert!(commands.is_empty());
15893    }
15894
15895    #[test]
15896    fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
15897        let nested_member = SelectionMemberMetadata {
15898            key: SelectionKey::Name("nested".to_string()),
15899            index: 0,
15900            base_sequence: 1,
15901            size: 2,
15902            kind: "group".to_string(),
15903        };
15904        let deadline_member = SelectionMemberMetadata {
15905            key: SelectionKey::Name("deadline".to_string()),
15906            index: 1,
15907            base_sequence: 3,
15908            size: 1,
15909            kind: "timer".to_string(),
15910        };
15911        let nested_paths = [
15912            vec![
15913                selection_group_entry(1, 3, 0, "mixed", &nested_member),
15914                parallel_group_entry(1, 2, 0, "activity"),
15915            ],
15916            vec![
15917                selection_group_entry(1, 3, 1, "mixed", &nested_member),
15918                parallel_group_entry(1, 2, 1, "activity"),
15919            ],
15920        ];
15921        let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
15922        let mut timer_fired = parallel_history_event(
15923            "TimerFired",
15924            3,
15925            "timer_id",
15926            "timer-3",
15927            deadline_path.clone(),
15928            None,
15929        );
15930        timer_fired.payload["delay_seconds"] = json!(0);
15931        timer_fired
15932            .raw
15933            .insert("id".to_string(), json!("timer-fired"));
15934        let mut timer_scheduled = parallel_history_event(
15935            "TimerScheduled",
15936            3,
15937            "timer_id",
15938            "timer-3",
15939            deadline_path,
15940            None,
15941        );
15942        timer_scheduled.payload["delay_seconds"] = json!(0);
15943        let history = vec![
15944            parallel_history_event(
15945                "ActivityScheduled",
15946                1,
15947                "activity_type",
15948                "nested-first",
15949                nested_paths[0].clone(),
15950                None,
15951            ),
15952            parallel_history_event(
15953                "ActivityScheduled",
15954                2,
15955                "activity_type",
15956                "nested-second",
15957                nested_paths[1].clone(),
15958                None,
15959            ),
15960            timer_scheduled,
15961            timer_fired,
15962            history_event(
15963                "SelectionResolved",
15964                json!({
15965                    "selection_group_id": "select-calls:1:3",
15966                    "selection_group_base_sequence": 1,
15967                    "selection_group_size": 3,
15968                    "member_key": "deadline",
15969                    "member_index": 1,
15970                    "member_base_sequence": 3,
15971                    "member_size": 1,
15972                    "operation_kind": "timer",
15973                    "operation_identity": "timer-3",
15974                    "outcome": "completed",
15975                    "resolution_event_id": "timer-fired",
15976                    "resolution_event_type": "TimerFired",
15977                }),
15978            ),
15979            parallel_history_event(
15980                "ActivityFailed",
15981                2,
15982                "activity_type",
15983                "nested-second",
15984                nested_paths[1].clone(),
15985                None,
15986            ),
15987        ];
15988        let ctx = workflow_context(history);
15989        let mut selection = Box::pin(ctx.select_keyed(vec![
15990            (
15991                "nested",
15992                ParallelOperation::group(vec![
15993                    ParallelOperation::activity("nested-first", json!([])),
15994                    ParallelOperation::activity("nested-second", json!([])),
15995                ]),
15996            ),
15997            ("deadline", ParallelOperation::timer(Duration::ZERO)),
15998        ]));
15999        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16000        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16001            panic!("deadline winner must replay");
16002        };
16003        let nested = selected
16004            .handle(&SelectionKey::Name("nested".to_string()))
16005            .expect("nested handle")
16006            .clone();
16007        let mut cancel = Box::pin(nested.cancel());
16008        assert!(matches!(
16009            cancel.as_mut().poll(&mut task_context),
16010            Poll::Ready(Ok(()))
16011        ));
16012        let mut await_nested = Box::pin(nested.await_result());
16013
16014        assert!(matches!(
16015            await_nested.as_mut().poll(&mut task_context),
16016            Poll::Ready(Err(Error::ActivityFailed(_)))
16017        ));
16018        assert!(ctx.take_commands().expect("commands").is_empty());
16019    }
16020
16021    #[test]
16022    fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16023        let ctx = workflow_context(Vec::new());
16024        let mut call = Box::pin(ctx.select(vec![
16025            ParallelOperation::child_workflow(
16026                "child",
16027                ChildWorkflowOptions::new("children"),
16028                json!([]),
16029            ),
16030            ParallelOperation::timer(Duration::from_secs(30)),
16031            ParallelOperation::signal("approval"),
16032            ParallelOperation::condition(
16033                ConditionWaitOptions::new("ready", "sha256:ready"),
16034                || Ok(false),
16035            ),
16036            ParallelOperation::group(vec![
16037                ParallelOperation::activity("nested-one", json!([])),
16038                ParallelOperation::activity("nested-two", json!([])),
16039            ]),
16040        ]));
16041        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16042        assert!(matches!(
16043            call.as_mut().poll(&mut task_context),
16044            Poll::Pending
16045        ));
16046        let commands = ctx.take_commands().expect("selection commands");
16047        assert_eq!(
16048            commands
16049                .iter()
16050                .map(|command| command["type"].as_str().unwrap_or_default())
16051                .collect::<Vec<_>>(),
16052            [
16053                "start_child_workflow",
16054                "start_timer",
16055                "open_signal_wait",
16056                "open_condition_wait",
16057                "schedule_activity",
16058                "schedule_activity",
16059            ]
16060        );
16061        assert!(commands.iter().all(|command| {
16062            command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16063        }));
16064        assert_eq!(
16065            commands[4]["parallel_group_path"].as_array().map(Vec::len),
16066            Some(2)
16067        );
16068        assert_eq!(
16069            commands[4]["parallel_group_path"][0]["selection_member_kind"],
16070            json!("group")
16071        );
16072        assert_eq!(
16073            commands[5]["parallel_group_path"][0]["selection_member_kind"],
16074            json!("group")
16075        );
16076
16077        let one_leaf_ctx = workflow_context(Vec::new());
16078        let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16079            ParallelOperation::activity("nested-only", json!([])),
16080        ])]));
16081        assert!(matches!(
16082            one_leaf.as_mut().poll(&mut task_context),
16083            Poll::Pending
16084        ));
16085        let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16086        assert_eq!(one_leaf_commands.len(), 1);
16087        assert_eq!(
16088            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16089            json!("group")
16090        );
16091        assert_eq!(
16092            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16093            json!(1)
16094        );
16095    }
16096
16097    async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16098        let mut saga = ctx.saga();
16099        let outcome = async {
16100            let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16101            saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16102            let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16103            saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16104            ctx.activity("trip.charge", json!([])).await?;
16105            Ok(json!({"status": "booked"}))
16106        }
16107        .await;
16108        saga.finish(outcome).await
16109    }
16110
16111    fn saga_activity(
16112        event_type: &str,
16113        sequence: u64,
16114        activity_type: &str,
16115        result: Option<Value>,
16116    ) -> HistoryEvent {
16117        let mut payload = json!({
16118            "sequence": sequence,
16119            "activity_type": activity_type,
16120            "message": format!("{activity_type} failed"),
16121            "exception_type": "PlannedFailure",
16122            "non_retryable": true,
16123        });
16124        if let Some(result) = result {
16125            payload["result"] = fixture_envelope(result);
16126        }
16127        history_event(event_type, payload)
16128    }
16129
16130    #[test]
16131    fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16132        let completed_hotel_compensation = saga_activity(
16133            "ActivityCompleted",
16134            4,
16135            "trip.cancel-hotel",
16136            Some(Value::Null),
16137        );
16138        let history = vec![
16139            saga_activity(
16140                "ActivityCompleted",
16141                1,
16142                "trip.reserve-flight",
16143                Some(json!("flight-1")),
16144            ),
16145            saga_activity(
16146                "ActivityCompleted",
16147                2,
16148                "trip.reserve-hotel",
16149                Some(json!("hotel-1")),
16150            ),
16151            saga_activity("ActivityFailed", 3, "trip.charge", None),
16152            completed_hotel_compensation.clone(),
16153            completed_hotel_compensation,
16154        ];
16155
16156        for _restart in 0..2 {
16157            let ctx = workflow_context(history.clone());
16158            let mut future = Box::pin(trip_saga(ctx.clone()));
16159            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16160            assert!(matches!(
16161                future.as_mut().poll(&mut task_context),
16162                Poll::Pending
16163            ));
16164            let commands = ctx.take_commands().expect("compensation command");
16165            assert_eq!(commands.len(), 1);
16166            assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16167        }
16168    }
16169
16170    #[test]
16171    fn saga_compensation_failure_preserves_both_typed_failures() {
16172        let history = vec![
16173            saga_activity(
16174                "ActivityCompleted",
16175                1,
16176                "trip.reserve-flight",
16177                Some(json!("flight-1")),
16178            ),
16179            saga_activity(
16180                "ActivityCompleted",
16181                2,
16182                "trip.reserve-hotel",
16183                Some(json!("hotel-1")),
16184            ),
16185            saga_activity("ActivityFailed", 3, "trip.charge", None),
16186            saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16187        ];
16188        let ctx = workflow_context(history);
16189        let mut future = Box::pin(trip_saga(ctx));
16190        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16191        let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16192            future.as_mut().poll(&mut task_context)
16193        else {
16194            panic!("compensation failure must remain structured");
16195        };
16196        assert!(matches!(
16197            *failure.initiating_failure,
16198            Error::ActivityFailed(_)
16199        ));
16200        assert!(matches!(
16201            *failure.compensation_failure,
16202            Error::ActivityFailed(_)
16203        ));
16204        assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16205        assert_eq!(failure.compensation_registration_order, 2);
16206    }
16207
16208    #[test]
16209    fn saga_compensates_cooperative_cancellation() {
16210        let ctx = workflow_context(vec![saga_activity(
16211            "ActivityCompleted",
16212            1,
16213            "trip.reserve-flight",
16214            Some(json!("flight-1")),
16215        )]);
16216        ctx.state.lock().expect("state").cancel_requested = true;
16217        let run = {
16218            let ctx = ctx.clone();
16219            async move {
16220                let mut saga = ctx.saga();
16221                let outcome = async {
16222                    let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16223                    saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16224                    ctx.throw_if_cancellation_requested()?;
16225                    Ok(json!("unexpected"))
16226                }
16227                .await;
16228                saga.finish(outcome).await
16229            }
16230        };
16231        let mut future = Box::pin(run);
16232        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16233        assert!(matches!(
16234            future.as_mut().poll(&mut task_context),
16235            Poll::Pending
16236        ));
16237        let commands = ctx.take_commands().expect("cancellation compensation");
16238        assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16239    }
16240
16241    fn workflow_task(
16242        workflow_type: &str,
16243        history_events: Vec<HistoryEvent>,
16244        payload_codec: &str,
16245    ) -> WorkflowTask {
16246        WorkflowTask {
16247            task_id: format!("wft-{workflow_type}"),
16248            workflow_command_id: None,
16249            workflow_id: Some(format!("wf-{workflow_type}")),
16250            run_id: Some(format!("run-{workflow_type}")),
16251            workflow_type: workflow_type.to_string(),
16252            cancel_requested: false,
16253            payload_codec: payload_codec.to_string(),
16254            arguments: Some(
16255                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16256            ),
16257            total_history_events: Some(history_events.len() as u64),
16258            history_size_bytes: None,
16259            continue_as_new_recommended: None,
16260            history_budget_pressure: None,
16261            history_events,
16262            next_history_page_token: None,
16263            workflow_task_attempt: 1,
16264            workflow_signal_id: None,
16265            signal_name: None,
16266            signal_arguments: None,
16267            workflow_update_id: None,
16268            update_name: None,
16269            lease_owner: Some("rust-worker".to_string()),
16270        }
16271    }
16272
16273    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16274    struct SideEffectProbe {
16275        request_id: String,
16276        attempt: u32,
16277    }
16278
16279    #[test]
16280    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16281        let calls = AtomicUsize::new(0);
16282        let ctx = workflow_context(Vec::new());
16283        let value = ctx
16284            .side_effect(|| {
16285                calls.fetch_add(1, Ordering::SeqCst);
16286                SideEffectProbe {
16287                    request_id: "request-42".to_string(),
16288                    attempt: 3,
16289                }
16290            })
16291            .expect("first side effect");
16292        assert_eq!(value.attempt, 3);
16293        assert_eq!(calls.load(Ordering::SeqCst), 1);
16294        let commands = ctx.take_commands().expect("commands");
16295        assert_eq!(commands.len(), 1);
16296        assert_eq!(commands[0]["type"], "record_side_effect");
16297        assert_eq!(
16298            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16299            serde_json::to_value(&value).expect("value")
16300        );
16301
16302        let replay = workflow_context(vec![history_event(
16303            "SideEffectRecorded",
16304            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16305        )]);
16306        let replayed: SideEffectProbe = replay
16307            .side_effect(|| {
16308                calls.fetch_add(1, Ordering::SeqCst);
16309                panic!("committed side-effect callbacks must not run during replay")
16310            })
16311            .expect("replayed side effect");
16312        assert_eq!(replayed, value);
16313        assert_eq!(calls.load(Ordering::SeqCst), 1);
16314        assert!(replay.take_commands().expect("commands").is_empty());
16315        replay.ensure_history_consumed().expect("history consumed");
16316    }
16317
16318    #[test]
16319    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16320        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16321        let value = ctx
16322            .side_effect(|| SideEffectProbe {
16323                request_id: "avro-request".to_string(),
16324                attempt: 1,
16325            })
16326            .expect("Avro side effect");
16327        let uuid = ctx.uuid_v4().expect("deterministic UUID");
16328        let commands = ctx.take_commands().expect("commands");
16329        assert_eq!(commands.len(), 2);
16330        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16331        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16332        assert_eq!(
16333            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16334            serde_json::to_value(&value).expect("value")
16335        );
16336
16337        let replay = workflow_context_with_codec(
16338            vec![
16339                history_event(
16340                    "SideEffectRecorded",
16341                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16342                ),
16343                history_event(
16344                    "SideEffectRecorded",
16345                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16346                ),
16347            ],
16348            DEFAULT_CODEC,
16349        );
16350        let replayed: SideEffectProbe = replay
16351            .side_effect(|| panic!("Avro callback must not run"))
16352            .expect("replayed Avro value");
16353        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16354        assert_eq!(replayed, value);
16355        assert_eq!(replayed_uuid, uuid);
16356        assert!(replay.take_commands().expect("commands").is_empty());
16357    }
16358
16359    #[test]
16360    fn typed_side_effect_replay_preserves_bytes_and_maps() {
16361        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16362        let value = ctx
16363            .side_effect_avro_value(typed_fidelity_probe)
16364            .expect("typed side effect");
16365        let commands = ctx.take_commands().expect("side-effect command");
16366        assert_eq!(
16367            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16368                .expect("recorded side effect"),
16369            value
16370        );
16371
16372        let replay = workflow_context_with_codec(
16373            vec![history_event(
16374                "SideEffectRecorded",
16375                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16376            )],
16377            DEFAULT_CODEC,
16378        );
16379        assert_eq!(
16380            replay
16381                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16382                .expect("replayed typed side effect"),
16383            value
16384        );
16385    }
16386
16387    #[test]
16388    fn ordered_side_effects_share_the_durable_command_stream() {
16389        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16390        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16391        let ctx = workflow_context(vec![
16392            history_event(
16393                "SideEffectRecorded",
16394                json!({"sequence": 1, "result": first}),
16395            ),
16396            history_event(
16397                "SideEffectRecorded",
16398                json!({"sequence": 2, "result": second}),
16399            ),
16400        ]);
16401        let first: String = ctx
16402            .side_effect(|| panic!("first callback must not run"))
16403            .expect("first replay");
16404        let second: i32 = ctx
16405            .side_effect(|| panic!("second callback must not run"))
16406            .expect("second replay");
16407        assert_eq!(first, "first");
16408        assert_eq!(second, 29);
16409        ctx.ensure_history_consumed().expect("ordered history");
16410
16411        let reordered = workflow_context(vec![history_event(
16412            "VersionMarkerRecorded",
16413            json!({
16414                "sequence": 1,
16415                "change_id": "before-side-effect",
16416                "version": 1,
16417                "min_supported": 1,
16418                "max_supported": 1,
16419            }),
16420        )]);
16421        let error = reordered
16422            .side_effect(|| "new".to_string())
16423            .expect_err("command reordering must fail");
16424        assert!(matches!(
16425            error,
16426            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16427                if reason == "recorded_command_mismatch"
16428        ));
16429    }
16430
16431    #[test]
16432    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16433        let ctx = workflow_context(Vec::new());
16434        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16435        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16436        assert!(ctx.patched("new-search").expect("patch"));
16437        ctx.deprecate_patch("new-search").expect("deprecate patch");
16438        let commands = ctx.take_commands().expect("commands");
16439        assert_eq!(commands.len(), 2);
16440        assert_eq!(commands[0]["type"], "record_version_marker");
16441        assert_eq!(commands[0]["version"], 2);
16442        assert_eq!(commands[1]["change_id"], "new-search");
16443
16444        let replay = workflow_context(vec![history_event(
16445            "VersionMarkerRecorded",
16446            json!({
16447                "sequence": 1,
16448                "change_id": "checkout-v2",
16449                "version": 2,
16450                "min_supported": 1,
16451                "max_supported": 2,
16452            }),
16453        )]);
16454        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16455        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16456        assert!(replay.take_commands().expect("commands").is_empty());
16457        replay.ensure_history_consumed().expect("history consumed");
16458    }
16459
16460    #[test]
16461    fn version_markers_reject_incompatible_or_malformed_history() {
16462        let incompatible = workflow_context(vec![history_event(
16463            "VersionMarkerRecorded",
16464            json!({
16465                "sequence": 1,
16466                "change_id": "checkout-v2",
16467                "version": 1,
16468                "min_supported": 1,
16469                "max_supported": 2,
16470            }),
16471        )]);
16472        let error = incompatible
16473            .get_version("checkout-v2", 2, 3)
16474            .expect_err("old version is unsupported");
16475        assert!(matches!(
16476            error,
16477            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16478                if reason == "version_marker_incompatible_range"
16479        ));
16480
16481        for (history, reason) in [
16482            (
16483                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16484                "side_effect_result_missing",
16485            ),
16486            (
16487                vec![history_event(
16488                    "SideEffectRecorded",
16489                    json!({
16490                        "sequence": 1,
16491                        "result": {"codec": "avro", "blob": "not-base64"},
16492                    }),
16493                )],
16494                "side_effect_payload_incompatible",
16495            ),
16496            (
16497                vec![history_event(
16498                    "SideEffectRecorded",
16499                    json!({"sequence": 1, "result": {"unwrapped": true}}),
16500                )],
16501                "side_effect_payload_malformed",
16502            ),
16503            (
16504                vec![history_event(
16505                    "VersionMarkerRecorded",
16506                    json!({
16507                        "sequence": 1,
16508                        "change_id": "change",
16509                        "version": 1,
16510                        "min_supported": 2,
16511                        "max_supported": 1,
16512                    }),
16513                )],
16514                "version_marker_history_range_invalid",
16515            ),
16516        ] {
16517            let error = WorkflowState::new(
16518                history,
16519                "rust-workers".to_string(),
16520                DEFAULT_CODEC.to_string(),
16521                None,
16522            )
16523            .expect_err("malformed history must fail");
16524            assert!(matches!(
16525                error,
16526                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16527                    if actual == reason
16528            ));
16529        }
16530    }
16531
16532    #[test]
16533    fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16534        let history = vec![history_event(
16535            "SearchAttributesUpserted",
16536            json!({
16537                "sequence": 1,
16538                "attributes": {"customer_tier": "gold"},
16539                "attribute_types": {"customer_tier": "keyword"},
16540                "merged": {"customer_tier": "gold"}
16541            }),
16542        )];
16543
16544        let matching = workflow_context(history.clone());
16545        matching
16546            .upsert_search_attributes(
16547                SearchAttributeUpdate::new()
16548                    .keyword("customer_tier", "gold")
16549                    .expect("keyword update"),
16550            )
16551            .expect("matching typed update must replay");
16552        matching
16553            .ensure_history_consumed()
16554            .expect("history consumed");
16555
16556        let changed_type = workflow_context(history.clone());
16557        let error = changed_type
16558            .upsert_search_attributes(
16559                SearchAttributeUpdate::new()
16560                    .string("customer_tier", "gold")
16561                    .expect("string update"),
16562            )
16563            .expect_err("same JSON value with a different declaration must be nondeterministic");
16564        let Error::NonDeterministicReplay(failure) = error else {
16565            panic!("typed identity drift must be a replay failure");
16566        };
16567        assert_eq!(failure.reason, "search_attribute_type_mismatch");
16568        assert_eq!(failure.sequence, Some(1));
16569
16570        let changed_value = workflow_context(history);
16571        let error = changed_value
16572            .upsert_search_attributes(
16573                SearchAttributeUpdate::new()
16574                    .keyword("customer_tier", "platinum")
16575                    .expect("keyword update"),
16576            )
16577            .expect_err("changed values must be nondeterministic");
16578        let Error::NonDeterministicReplay(failure) = error else {
16579            panic!("value drift must be a replay failure");
16580        };
16581        assert_eq!(failure.reason, "search_attribute_value_mismatch");
16582    }
16583
16584    #[test]
16585    fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16586        let history = vec![history_event(
16587            "SearchAttributesUpserted",
16588            json!({
16589                "sequence": 1,
16590                "attributes": {"customer_tier": "gold"},
16591                "merged": {"customer_tier": "gold"}
16592            }),
16593        )];
16594
16595        for update in [
16596            SearchAttributeUpdate::new()
16597                .keyword("customer_tier", "gold")
16598                .expect("keyword update"),
16599            SearchAttributeUpdate::new()
16600                .string("customer_tier", "gold")
16601                .expect("string update"),
16602        ] {
16603            let restarted = workflow_context(history.clone());
16604            restarted
16605                .upsert_search_attributes(update)
16606                .expect("legacy history constrains values but has unknown type identity");
16607            restarted
16608                .ensure_history_consumed()
16609                .expect("history consumed");
16610        }
16611    }
16612
16613    #[test]
16614    fn search_attribute_command_emits_canonical_types() {
16615        let ctx = workflow_context(Vec::new());
16616        ctx.upsert_search_attributes(
16617            SearchAttributeUpdate::new()
16618                .keyword("customer_tier", "gold")
16619                .expect("keyword update")
16620                .int("attempts", 3)
16621                .expect("int update")
16622                .delete("obsolete")
16623                .expect("delete update"),
16624        )
16625        .expect("valid search attributes");
16626
16627        assert_eq!(
16628            ctx.take_commands().expect("commands"),
16629            vec![json!({
16630                "type": "upsert_search_attributes",
16631                "attributes": {
16632                    "attempts": 3,
16633                    "customer_tier": "gold",
16634                    "obsolete": null
16635                },
16636                "attribute_types": {
16637                    "attempts": "int",
16638                    "customer_tier": "keyword"
16639                }
16640            })]
16641        );
16642    }
16643
16644    #[test]
16645    fn duplicate_side_effects_and_version_markers_are_rejected() {
16646        let duplicate_side_effect = WorkflowState::new(
16647            vec![
16648                history_event(
16649                    "SideEffectRecorded",
16650                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16651                ),
16652                history_event(
16653                    "SideEffectRecorded",
16654                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16655                ),
16656            ],
16657            "rust-workers".to_string(),
16658            DEFAULT_CODEC.to_string(),
16659            None,
16660        )
16661        .expect_err("duplicate side effect");
16662        assert!(matches!(
16663            duplicate_side_effect,
16664            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16665                if reason == "duplicate_side_effect_record"
16666        ));
16667
16668        let marker = |sequence| {
16669            history_event(
16670                "VersionMarkerRecorded",
16671                json!({
16672                    "sequence": sequence,
16673                    "change_id": "same-change",
16674                    "version": 1,
16675                    "min_supported": 1,
16676                    "max_supported": 1,
16677                }),
16678            )
16679        };
16680        let duplicate_marker = WorkflowState::new(
16681            vec![marker(1), marker(3)],
16682            "rust-workers".to_string(),
16683            DEFAULT_CODEC.to_string(),
16684            None,
16685        )
16686        .expect_err("duplicate marker");
16687        assert!(matches!(
16688            duplicate_marker,
16689            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16690                if reason == "duplicate_version_marker"
16691        ));
16692    }
16693
16694    #[test]
16695    fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16696        let mut state = WorkflowState::new(
16697            Vec::new(),
16698            "rust-workers".to_string(),
16699            DEFAULT_CODEC.to_string(),
16700            None,
16701        )
16702        .expect("workflow state");
16703        state.workflow_command_identity = "command-7".to_string();
16704        let context = WorkflowContext {
16705            state: Arc::new(Mutex::new(state)),
16706        };
16707        let item =
16708            WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16709
16710        context
16711            .append_workflow_stream("output", &[item], Some(10))
16712            .expect("append command");
16713        context
16714            .error_workflow_stream("output", "producer failed", None)
16715            .expect("error command");
16716        let commands = context.take_commands().expect("commands");
16717
16718        assert_eq!(commands[0]["type"], "record_side_effect");
16719        assert_eq!(
16720            commands[0]["workflow_stream"]["command_identity"],
16721            "command-7"
16722        );
16723        assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16724        assert_eq!(
16725            commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16726            "dw-stream:command-7:0:0"
16727        );
16728        assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16729
16730        let recorded = history_event(
16731            "SideEffectRecorded",
16732            json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16733        );
16734        let mut replay_state = WorkflowState::new(
16735            vec![recorded],
16736            "rust-workers".to_string(),
16737            DEFAULT_CODEC.to_string(),
16738            None,
16739        )
16740        .expect("replay state");
16741        replay_state.workflow_command_identity = "command-7".to_string();
16742        let replay_context = WorkflowContext {
16743            state: Arc::new(Mutex::new(replay_state)),
16744        };
16745        replay_context
16746            .append_workflow_stream(
16747                "output",
16748                &[WorkflowStreamAppendItem::from_reference(
16749                    "s3://bucket/item.avro",
16750                )],
16751                Some(10),
16752            )
16753            .expect("replayed append");
16754        assert!(replay_context
16755            .take_commands()
16756            .expect("replayed commands")
16757            .is_empty());
16758    }
16759
16760    #[test]
16761    fn workflow_stream_authoring_requires_server_durable_command_identity() {
16762        let context = workflow_context(Vec::new());
16763        let error = context
16764            .append_workflow_stream(
16765                "output",
16766                &[WorkflowStreamAppendItem::from_reference(
16767                    "s3://bucket/item.avro",
16768                )],
16769                None,
16770            )
16771            .expect_err("stream append without durable command identity must fail closed");
16772
16773        assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16774        assert!(context.take_commands().expect("commands").is_empty());
16775    }
16776
16777    #[test]
16778    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16779        fn worker(calls: Arc<AtomicUsize>) -> Worker {
16780            let client = Client::new("http://127.0.0.1:8080").expect("client");
16781            let mut worker = Worker::new(client, "rust-workers");
16782            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16783                let calls = Arc::clone(&calls);
16784                async move {
16785                    let captured = ctx.side_effect(|| {
16786                        calls.fetch_add(1, Ordering::SeqCst);
16787                        "captured-once".to_string()
16788                    })?;
16789                    let version = ctx.get_version("cold-restart", 1, 2)?;
16790                    Ok(json!({"captured": captured, "version": version}))
16791                }
16792            });
16793            worker
16794        }
16795
16796        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16797            WorkflowTask {
16798                task_id: "wft-side-effect-version".to_string(),
16799                workflow_command_id: None,
16800                workflow_id: Some("wf-side-effect-version".to_string()),
16801                run_id: Some("run-side-effect-version".to_string()),
16802                workflow_type: "rust.side-effect-version".to_string(),
16803                cancel_requested: false,
16804                payload_codec: DEFAULT_CODEC.to_string(),
16805                arguments: Some(
16806                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16807                ),
16808                history_events,
16809                total_history_events: None,
16810                history_size_bytes: None,
16811                continue_as_new_recommended: None,
16812                history_budget_pressure: None,
16813                next_history_page_token: None,
16814                workflow_task_attempt: 1,
16815                workflow_signal_id: None,
16816                signal_name: None,
16817                signal_arguments: None,
16818                workflow_update_id: None,
16819                update_name: None,
16820                lease_owner: Some("rust-worker".to_string()),
16821            }
16822        }
16823
16824        let calls = Arc::new(AtomicUsize::new(0));
16825        let initial = worker(Arc::clone(&calls))
16826            .execute_workflow_task(task(Vec::new()))
16827            .expect("initial execution");
16828        assert_eq!(
16829            initial
16830                .iter()
16831                .map(|command| &command["type"])
16832                .collect::<Vec<_>>(),
16833            vec![
16834                "record_side_effect",
16835                "record_version_marker",
16836                "complete_workflow"
16837            ]
16838        );
16839        assert_eq!(calls.load(Ordering::SeqCst), 1);
16840
16841        let restarted = worker(Arc::clone(&calls));
16842        let replayed = restarted
16843            .execute_workflow_task(task(vec![
16844                history_event(
16845                    "SideEffectRecorded",
16846                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16847                ),
16848                history_event(
16849                    "VersionMarkerRecorded",
16850                    json!({
16851                        "sequence": 2,
16852                        "change_id": "cold-restart",
16853                        "version": 2,
16854                        "min_supported": 1,
16855                        "max_supported": 2,
16856                    }),
16857                ),
16858            ]))
16859            .expect("cold replay");
16860        assert_eq!(replayed.len(), 1);
16861        assert_eq!(replayed[0]["type"], "complete_workflow");
16862        assert_eq!(calls.load(Ordering::SeqCst), 1);
16863    }
16864
16865    #[test]
16866    fn side_effect_replay_rejects_changed_rust_value_type() {
16867        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16868        let ctx = workflow_context(vec![history_event(
16869            "SideEffectRecorded",
16870            json!({"sequence": 1, "result": result}),
16871        )]);
16872        let error = ctx
16873            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16874            .expect_err("changed type must fail replay");
16875        assert!(matches!(
16876            error,
16877            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16878                if reason == "side_effect_type_mismatch"
16879        ));
16880    }
16881
16882    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16883        vec![
16884            history_event(
16885                "ActivityScheduled",
16886                json!({
16887                    "sequence": 1,
16888                    "activity_type": "flaky",
16889                    "activity_execution_id": "act-1",
16890                    "activity": {
16891                        "id": "act-1",
16892                        "sequence": 1,
16893                        "type": "flaky",
16894                        "queue": "critical-activities",
16895                        "execution_mode": null,
16896                        "retry_policy": {
16897                            "snapshot_version": 1,
16898                            "max_attempts": 3,
16899                            "backoff_seconds": [2, 4],
16900                            "start_to_close_timeout": 30,
16901                            "schedule_to_start_timeout": 5,
16902                            "schedule_to_close_timeout": 90,
16903                            "heartbeat_timeout": 10,
16904                            "non_retryable_error_types": ["PermanentError"]
16905                        }
16906                    }
16907                }),
16908            ),
16909            history_event(
16910                "ActivityStarted",
16911                json!({
16912                    "sequence": 1,
16913                    "activity_type": "flaky",
16914                    "activity_execution_id": "act-1",
16915                    "activity_attempt_id": "attempt-1",
16916                    "attempt_number": 1
16917                }),
16918            ),
16919            history_event(
16920                "ActivityRetryScheduled",
16921                json!({
16922                    "sequence": 1,
16923                    "activity_type": "flaky",
16924                    "activity_execution_id": "act-1",
16925                    "activity_attempt_id": "attempt-1",
16926                    "attempt_number": 1,
16927                    "retry_after_attempt": 1,
16928                    "retry_backoff_seconds": 2,
16929                    "failure_category": "activity",
16930                    "exception_type": "TransientError"
16931                }),
16932            ),
16933            history_event(
16934                "ActivityStarted",
16935                json!({
16936                    "sequence": 1,
16937                    "activity_type": "flaky",
16938                    "activity_execution_id": "act-1",
16939                    "activity_attempt_id": "attempt-2",
16940                    "attempt_number": 2
16941                }),
16942            ),
16943            history_event(
16944                "ActivityCompleted",
16945                json!({
16946                    "sequence": 1,
16947                    "activity_type": "flaky",
16948                    "activity_execution_id": "act-1",
16949                    "activity_attempt_id": "attempt-2",
16950                    "attempt_number": 2,
16951                    "payload_codec": DEFAULT_CODEC,
16952                    "result": fixture_envelope(json!({"status":"recovered"}))
16953                }),
16954            ),
16955        ]
16956    }
16957
16958    fn retry_activity_options() -> ActivityOptions {
16959        ActivityOptions::new()
16960            .task_queue("critical-activities")
16961            .retry_policy(
16962                ActivityRetryPolicy::new(3)
16963                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
16964                    .non_retryable_error_type("PermanentError"),
16965            )
16966            .start_to_close_timeout(Duration::from_secs(30))
16967            .schedule_to_start_timeout(Duration::from_secs(5))
16968            .schedule_to_close_timeout(Duration::from_secs(90))
16969            .heartbeat_timeout(Duration::from_secs(10))
16970    }
16971
16972    #[test]
16973    fn fixed_avro_value_round_trips_json_values() {
16974        let value = json!({"greeting": "hello", "count": 3, "ok": true});
16975        let envelope = PayloadEnvelope::avro(&value).expect("encode");
16976        assert_eq!(envelope.codec, DEFAULT_CODEC);
16977        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
16978    }
16979
16980    #[tokio::test]
16981    async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
16982        let client = Client::new("http://127.0.0.1:8080").expect("client");
16983        let mut worker = Worker::new(client, "rust-workers");
16984        worker.register_typed_workflow(
16985            "typed.contract.workflow",
16986            |_ctx, input: TypedContract| async move { Ok(input) },
16987        );
16988        worker.register_typed_activity(
16989            "typed.contract.activity",
16990            |_ctx, input: TypedContract| async move { Ok(input) },
16991        );
16992
16993        let expected = typed_contract();
16994        let arguments = AvroValue::Array(vec![
16995            AvroValue::from_serialize(&expected).expect("typed request")
16996        ]);
16997        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
16998        let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
16999        workflow.arguments = Some(envelope.clone());
17000        let commands = worker
17001            .execute_workflow_task(workflow)
17002            .expect("typed workflow task");
17003        let workflow_result: TypedContract =
17004            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17005                .expect("workflow result envelope")
17006                .deserialize()
17007                .expect("workflow result type");
17008        assert_eq!(workflow_result, expected);
17009
17010        let activity = ActivityTask {
17011            task_id: "typed-contract-activity".to_string(),
17012            activity_attempt_id: Some("typed-contract-attempt".to_string()),
17013            attempt_id: None,
17014            activity_type: "typed.contract.activity".to_string(),
17015            payload_codec: DEFAULT_CODEC.to_string(),
17016            arguments: Some(envelope),
17017            attempt_number: 1,
17018            lease_owner: Some("rust-worker".to_string()),
17019        };
17020        let activity_result: TypedContract = worker
17021            .execute_activity_task(activity)
17022            .await
17023            .expect("typed activity task")
17024            .deserialize()
17025            .expect("activity result type");
17026        assert_eq!(activity_result, expected);
17027    }
17028
17029    #[tokio::test]
17030    async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17031        let client = Client::new("http://127.0.0.1:8080").expect("client");
17032        let mut worker = Worker::new(client, "rust-workers");
17033        worker.register_typed_workflow(
17034            "typed.shape.workflow",
17035            |_ctx, input: TypedContract| async move { Ok(input) },
17036        );
17037        worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17038            Ok(f64::NAN)
17039        });
17040
17041        let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17042        workflow.arguments = Some(
17043            encode_typed_envelope(
17044                &AvroValue::Array(vec![
17045                    AvroValue::String("first".to_string()),
17046                    AvroValue::String("second".to_string()),
17047                ]),
17048                DEFAULT_CODEC,
17049            )
17050            .expect("malformed typed arguments"),
17051        );
17052        let commands = worker
17053            .execute_workflow_task(workflow)
17054            .expect("shape mismatch becomes a workflow failure");
17055        let message = commands[0]["message"].as_str().expect("failure message");
17056        assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17057        assert!(message.contains(type_name::<TypedContract>()));
17058        assert!(message.contains("task carried 2 arguments"));
17059
17060        let activity = ActivityTask {
17061            task_id: "typed-unsupported-activity".to_string(),
17062            activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17063            attempt_id: None,
17064            activity_type: "typed.unsupported.activity".to_string(),
17065            payload_codec: DEFAULT_CODEC.to_string(),
17066            arguments: Some(
17067                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17068                    .expect("unit arguments"),
17069            ),
17070            attempt_number: 1,
17071            lease_owner: Some("rust-worker".to_string()),
17072        };
17073        let Error::HandlerType {
17074            handler_kind,
17075            handler_name,
17076            value_kind,
17077            rust_type,
17078            message,
17079        } = worker
17080            .execute_activity_task(activity)
17081            .await
17082            .expect_err("non-finite handler output must fail")
17083        else {
17084            panic!("expected contextual handler type failure");
17085        };
17086        assert_eq!(handler_kind, HandlerKind::Activity);
17087        assert_eq!(handler_name, "typed.unsupported.activity");
17088        assert_eq!(value_kind, HandlerValueKind::Result);
17089        assert_eq!(rust_type, type_name::<f64>());
17090        assert!(message.contains("non_finite_float"));
17091    }
17092
17093    #[tokio::test]
17094    async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17095        #[derive(Clone, Default)]
17096        struct State {
17097            observed: Option<TypedContract>,
17098        }
17099
17100        let client = Client::new("http://127.0.0.1:8080").expect("client");
17101        let mut worker = Worker::new(client, "rust-workers");
17102        worker.register_typed_replayed_workflow(
17103            "typed.contract.replayed",
17104            State::default,
17105            |ctx, input: TypedContract, state| async move {
17106                let result: TypedContract =
17107                    ctx.activity_typed("typed.contract.activity", input).await?;
17108                state.update(|current| current.observed = Some(result.clone()))?;
17109                Ok(result)
17110            },
17111        );
17112        worker.register_replayed_query::<State, _, _>(
17113            "typed.contract.replayed",
17114            "observed",
17115            |_ctx, state, _args| async move {
17116                Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17117            },
17118        );
17119
17120        let expected = typed_contract();
17121        let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17122        let workflow_arguments =
17123            encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17124                .expect("workflow arguments");
17125        let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17126        let task = QueryTask {
17127            query_task_id: "typed-replay-query".to_string(),
17128            query_task_attempt: 1,
17129            lease_owner: Some("rust-worker".to_string()),
17130            workflow_id: Some("typed-replay".to_string()),
17131            run_id: Some("typed-replay-run".to_string()),
17132            workflow_type: "typed.contract.replayed".to_string(),
17133            query_name: "observed".to_string(),
17134            payload_codec: DEFAULT_CODEC.to_string(),
17135            workflow_arguments: Some(workflow_arguments),
17136            query_arguments: Some(
17137                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17138                    .expect("query arguments"),
17139            ),
17140            history_events: vec![
17141                history_event(
17142                    "ActivityScheduled",
17143                    json!({
17144                        "sequence": 1,
17145                        "activity_type": "typed.contract.activity"
17146                    }),
17147                ),
17148                history_event(
17149                    "ActivityCompleted",
17150                    json!({
17151                        "sequence": 1,
17152                        "activity_type": "typed.contract.activity",
17153                        "payload_codec": DEFAULT_CODEC,
17154                        "result": result
17155                    }),
17156                ),
17157            ],
17158            history_export: None,
17159            run_status: Some("completed".to_string()),
17160        };
17161
17162        assert_eq!(
17163            worker
17164                .execute_query_task(task)
17165                .await
17166                .expect("typed replay query")
17167                .deserialize::<i64>()
17168                .expect("query result"),
17169            expected.signed
17170        );
17171    }
17172
17173    #[tokio::test]
17174    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17175        let client = Client::new("http://127.0.0.1:8080").expect("client");
17176        let mut worker = Worker::new(client, "rust-workers");
17177        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17178        worker
17179            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17180        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17181            Ok(input)
17182        });
17183        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17184            Ok(input)
17185        });
17186        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17187            Ok(AvroValue::Array(
17188                ctx.wait_signal_avro_value("changed").await?,
17189            ))
17190        });
17191
17192        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17193        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17194
17195        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17196        workflow.arguments = Some(envelope.clone());
17197        let commands = worker
17198            .execute_workflow_task(workflow)
17199            .expect("typed workflow task");
17200        assert_eq!(commands[0]["type"], "complete_workflow");
17201        assert_eq!(
17202            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17203                .expect("typed workflow result"),
17204            arguments
17205        );
17206
17207        let activity = ActivityTask {
17208            task_id: "activity-typed".to_string(),
17209            activity_attempt_id: Some("attempt-typed".to_string()),
17210            attempt_id: None,
17211            activity_type: "typed.activity".to_string(),
17212            payload_codec: DEFAULT_CODEC.to_string(),
17213            arguments: Some(envelope.clone()),
17214            attempt_number: 1,
17215            lease_owner: Some("rust-worker".to_string()),
17216        };
17217        assert_eq!(
17218            worker
17219                .execute_activity_task(activity)
17220                .await
17221                .expect("typed activity result"),
17222            arguments
17223        );
17224
17225        let query = QueryTask {
17226            query_task_id: "query-typed".to_string(),
17227            query_task_attempt: 1,
17228            lease_owner: Some("rust-worker".to_string()),
17229            workflow_id: Some("typed-1".to_string()),
17230            run_id: Some("run-typed".to_string()),
17231            workflow_type: "typed.echo".to_string(),
17232            query_name: "inspect".to_string(),
17233            payload_codec: DEFAULT_CODEC.to_string(),
17234            workflow_arguments: Some(
17235                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17236                    .expect("workflow input"),
17237            ),
17238            query_arguments: Some(envelope.clone()),
17239            history_events: Vec::new(),
17240            history_export: None,
17241            run_status: Some("running".to_string()),
17242        };
17243        assert_eq!(
17244            worker
17245                .execute_query_task(query)
17246                .await
17247                .expect("typed query result"),
17248            arguments
17249        );
17250
17251        let mut update = workflow_task(
17252            "typed.echo",
17253            vec![history_event(
17254                "UpdateAccepted",
17255                json!({
17256                    "update_id": "update-typed",
17257                    "update_name": "replace",
17258                    "arguments": envelope.clone(),
17259                }),
17260            )],
17261            DEFAULT_CODEC,
17262        );
17263        update.workflow_update_id = Some("update-typed".to_string());
17264        update.update_name = Some("replace".to_string());
17265        let commands = worker
17266            .execute_workflow_task(update)
17267            .expect("typed update task");
17268        assert_eq!(commands[0]["type"], "complete_update");
17269        assert_eq!(
17270            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17271                .expect("typed update result"),
17272            arguments
17273        );
17274
17275        let mut signal = workflow_task(
17276            "typed.signal",
17277            vec![history_event(
17278                "SignalReceived",
17279                json!({
17280                    "signal_id": "signal-typed",
17281                    "signal_name": "changed",
17282                    "arguments": envelope.clone(),
17283                }),
17284            )],
17285            DEFAULT_CODEC,
17286        );
17287        signal.workflow_signal_id = Some("signal-typed".to_string());
17288        signal.signal_name = Some("changed".to_string());
17289        signal.signal_arguments = Some(envelope);
17290        let commands = worker
17291            .execute_workflow_task(signal)
17292            .expect("typed signal resume");
17293        assert_eq!(
17294            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17295                .expect("typed signal result"),
17296            arguments
17297        );
17298    }
17299
17300    #[tokio::test]
17301    async fn typed_helpers_never_parse_json_inspection_projection() {
17302        let collision_values = projection_collision_probe();
17303        let expected = AvroValue::Array(collision_values.clone());
17304        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17305
17306        let activity_context = workflow_context_with_codec(
17307            vec![history_event(
17308                "ActivityCompleted",
17309                json!({
17310                    "sequence": 1,
17311                    "activity_type": "collision.activity",
17312                    "payload_codec": DEFAULT_CODEC,
17313                    "result": envelope.clone(),
17314                }),
17315            )],
17316            DEFAULT_CODEC,
17317        );
17318        assert_eq!(
17319            activity_context
17320                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17321                .await
17322                .expect("typed activity collision result"),
17323            expected
17324        );
17325
17326        let signal_context = workflow_context_with_codec(
17327            vec![
17328                history_event(
17329                    "SignalWaitOpened",
17330                    json!({"sequence": 1, "signal_name": "collision"}),
17331                ),
17332                history_event(
17333                    "SignalApplied",
17334                    json!({
17335                        "sequence": 1,
17336                        "signal_name": "collision",
17337                        "payload_codec": DEFAULT_CODEC,
17338                        "value": envelope.clone(),
17339                    }),
17340                ),
17341            ],
17342            DEFAULT_CODEC,
17343        );
17344        assert_eq!(
17345            signal_context
17346                .wait_signal_avro_value("collision")
17347                .await
17348                .expect("typed signal collision arguments"),
17349            collision_values
17350        );
17351
17352        let child_context = workflow_context_with_codec(
17353            vec![
17354                history_event(
17355                    "ChildWorkflowScheduled",
17356                    json!({
17357                        "sequence": 1,
17358                        "child_workflow_instance_id": "collision-child",
17359                        "child_workflow_run_id": "collision-run",
17360                        "child_workflow_type": "collision.child",
17361                    }),
17362                ),
17363                history_event(
17364                    "ChildRunCompleted",
17365                    json!({
17366                        "sequence": 1,
17367                        "child_workflow_instance_id": "collision-child",
17368                        "child_workflow_run_id": "collision-run",
17369                        "child_workflow_type": "collision.child",
17370                        "payload_codec": DEFAULT_CODEC,
17371                        "result": envelope,
17372                    }),
17373                ),
17374            ],
17375            DEFAULT_CODEC,
17376        );
17377        let child = child_context
17378            .start_child_workflow_avro_value(
17379                "collision.child",
17380                ChildWorkflowOptions::new("collision-workers"),
17381                AvroValue::Array(Vec::new()),
17382            )
17383            .await
17384            .expect("typed child collision result");
17385        assert_eq!(child.result, expected);
17386    }
17387
17388    #[tokio::test]
17389    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17390        let client = Client::new("http://127.0.0.1:8080").expect("client");
17391        let mut worker = Worker::new(client, "rust-workers");
17392        worker.register_replayed_workflow_avro_value(
17393            "typed.replayed",
17394            || (),
17395            |_ctx, input, _state| async move { Ok(input) },
17396        );
17397        worker.register_replayed_query_avro_value::<(), _, _>(
17398            "typed.replayed",
17399            "inspect",
17400            |ctx, _state, args| async move {
17401                let mut signals = ctx.signals_avro_value("collision");
17402                let signal = signals
17403                    .pop()
17404                    .map(AvroValue::Array)
17405                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17406                Ok(AvroValue::Array(vec![
17407                    ctx.workflow_input_avro_value().clone(),
17408                    signal,
17409                    args,
17410                ]))
17411            },
17412        );
17413        let arguments = AvroValue::Array(projection_collision_probe());
17414        let signal_arguments =
17415            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17416        let task = QueryTask {
17417            query_task_id: "query-typed-replay".to_string(),
17418            query_task_attempt: 1,
17419            lease_owner: Some("rust-worker".to_string()),
17420            workflow_id: Some("typed-replay".to_string()),
17421            run_id: Some("run-typed-replay".to_string()),
17422            workflow_type: "typed.replayed".to_string(),
17423            query_name: "inspect".to_string(),
17424            payload_codec: DEFAULT_CODEC.to_string(),
17425            workflow_arguments: Some(
17426                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17427            ),
17428            query_arguments: Some(
17429                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17430            ),
17431            history_events: vec![history_event(
17432                "SignalReceived",
17433                json!({
17434                    "signal_id": "collision-signal",
17435                    "signal_name": "collision",
17436                    "workflow_sequence": 1,
17437                    "payload_codec": DEFAULT_CODEC,
17438                    "arguments": signal_arguments,
17439                }),
17440            )],
17441            history_export: None,
17442            run_status: Some("completed".to_string()),
17443        };
17444
17445        assert_eq!(
17446            worker
17447                .execute_query_task(task)
17448                .await
17449                .expect("typed replay query"),
17450            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17451        );
17452    }
17453
17454    #[test]
17455    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17456        let value = BTreeMap::from([(1_i32, "integer key")]);
17457        let error = PayloadEnvelope::avro(&value)
17458            .expect_err("integer map keys must fail")
17459            .to_string();
17460
17461        assert!(error.contains("invalid_map_key"));
17462    }
17463
17464    #[test]
17465    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17466        let envelope = PayloadEnvelope {
17467            codec: "json".to_string(),
17468            blob: r#"{"greeting":"hello"}"#.to_string(),
17469        };
17470
17471        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17472        let diagnostic = error.to_string();
17473        assert!(diagnostic.contains("unsupported_payload_codec"));
17474        assert!(diagnostic.contains("codec=\"avro\""));
17475        assert!(diagnostic.contains("HTTP document transport"));
17476    }
17477
17478    #[test]
17479    fn untagged_json_payload_value_fails_closed() {
17480        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17481            .expect_err("untagged JSON payload values must fail");
17482        let diagnostic = error.to_string();
17483        assert!(diagnostic.contains("unsupported_payload_codec"));
17484        assert!(diagnostic.contains("untagged durable payload"));
17485        assert!(diagnostic.contains("HTTP document transport"));
17486    }
17487
17488    #[test]
17489    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17490        let envelope = PayloadEnvelope {
17491            codec: DEFAULT_CODEC.to_string(),
17492            blob: BASE64.encode([0x01]),
17493        };
17494
17495        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17496        assert!(error.to_string().contains("invalid_payload_framing"));
17497    }
17498
17499    #[tokio::test]
17500    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17501        let server = MockWorkerServer::start();
17502        let client = Client::builder(server.base_url())
17503            .timeout(Duration::from_secs(2))
17504            .build()
17505            .expect("client");
17506        let invalid_commands = [
17507            json!({
17508                "type": "complete_workflow",
17509                "result": {"codec": "json", "blob": null}
17510            }),
17511            json!({
17512                "type": "schedule_activity",
17513                "arguments": {"codec": "yaml", "blob": "ignored"}
17514            }),
17515            json!({
17516                "type": "start_child_workflow",
17517                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17518            }),
17519            json!({"type": "continue_as_new", "arguments": []}),
17520            json!({"type": "complete_update"}),
17521            json!({"type": "record_side_effect", "result": null}),
17522            json!({
17523                "type": "start_service_operation",
17524                "payload_codec": DEFAULT_CODEC,
17525                "request_payload": "raw-avro-bytes"
17526            }),
17527        ];
17528
17529        for command in invalid_commands {
17530            let error = client
17531                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17532                .await
17533                .expect_err("invalid durable payload must fail locally");
17534            let diagnostic = error.to_string();
17535            assert!(
17536                diagnostic.contains("unsupported_payload_codec")
17537                    || diagnostic.contains("invalid_payload_envelope")
17538                    || diagnostic.contains("untagged durable payload"),
17539                "unexpected validation diagnostic: {diagnostic}"
17540            );
17541        }
17542
17543        assert_eq!(
17544            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17545            0,
17546            "invalid command payloads must not reach HTTP transport"
17547        );
17548    }
17549
17550    #[test]
17551    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17552        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17553        let commands = [
17554            json!({"type": "complete_workflow", "result": envelope.clone()}),
17555            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17556            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17557            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17558            json!({"type": "complete_update", "result": envelope.clone()}),
17559            json!({"type": "record_side_effect", "result": envelope.clone()}),
17560            json!({
17561                "type": "start_service_operation",
17562                "payload_codec": DEFAULT_CODEC,
17563                "request_payload": envelope.clone()
17564            }),
17565            json!({
17566                "type": "complete_workflow",
17567                "result": envelope,
17568                "metadata": {
17569                    "codec": "json",
17570                    "payload_codec": "customer-codec",
17571                    "result": {"codec": "yaml", "blob": null}
17572                }
17573            }),
17574        ];
17575
17576        validate_workflow_task_commands(&commands)
17577            .expect("customer metadata must not become a protocol codec declaration");
17578    }
17579
17580    #[test]
17581    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17582        assert_eq!(
17583            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17584            AvroValue::Array(Vec::new())
17585        );
17586        assert_eq!(
17587            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17588            AvroValue::Array(Vec::new())
17589        );
17590
17591        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17592        signal.signal_name = Some("empty-signal".to_string());
17593        signal.signal_arguments = None;
17594        let decoded = decode_resume_signal(&signal)
17595            .expect("valid Avro signal")
17596            .expect("named signal resumes the workflow");
17597        assert!(decoded.arguments.is_empty());
17598    }
17599
17600    #[tokio::test]
17601    async fn malformed_task_level_codecs_become_pre_handler_failures() {
17602        let client = Client::new("http://127.0.0.1:8080").expect("client");
17603        let mut worker = Worker::new(client, "rust-workers");
17604        let handler_calls = Arc::new(AtomicUsize::new(0));
17605
17606        let calls = Arc::clone(&handler_calls);
17607        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17608            calls.fetch_add(1, Ordering::SeqCst);
17609            async move { Ok(Value::Null) }
17610        });
17611        let calls = Arc::clone(&handler_calls);
17612        worker.register_activity("codec.activity", move |_ctx, _args| {
17613            calls.fetch_add(1, Ordering::SeqCst);
17614            async move { Ok(Value::Null) }
17615        });
17616        let calls = Arc::clone(&handler_calls);
17617        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17618            calls.fetch_add(1, Ordering::SeqCst);
17619            async move { Ok(Value::Null) }
17620        });
17621
17622        let mut failures = Vec::new();
17623        for codec_case in [
17624            InvalidTaskPayloadCodec::Missing,
17625            InvalidTaskPayloadCodec::Null,
17626            InvalidTaskPayloadCodec::NonString,
17627        ] {
17628            let mut workflow = json!({
17629                "task_id": format!("workflow-{}", codec_case.label()),
17630                "workflow_type": "codec.workflow"
17631            });
17632            codec_case.apply(&mut workflow);
17633            match serde_json::from_value::<WorkflowTask>(workflow) {
17634                Ok(task) => match worker.execute_workflow_task(task) {
17635                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17636                    outcome => failures.push(format!(
17637                        "workflow {} codec returned {outcome:?}",
17638                        codec_case.label()
17639                    )),
17640                },
17641                Err(error) => failures.push(format!(
17642                    "workflow {} codec failed transport deserialization: {error}",
17643                    codec_case.label()
17644                )),
17645            }
17646
17647            let mut activity = json!({
17648                "task_id": format!("activity-{}", codec_case.label()),
17649                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17650                "activity_type": "codec.activity",
17651                "attempt_number": 1
17652            });
17653            codec_case.apply(&mut activity);
17654            match serde_json::from_value::<ActivityTask>(activity) {
17655                Ok(task) => match worker.execute_activity_task(task).await {
17656                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17657                    outcome => failures.push(format!(
17658                        "activity {} codec returned {outcome:?}",
17659                        codec_case.label()
17660                    )),
17661                },
17662                Err(error) => failures.push(format!(
17663                    "activity {} codec failed transport deserialization: {error}",
17664                    codec_case.label()
17665                )),
17666            }
17667
17668            let mut query = json!({
17669                "query_task_id": format!("query-{}", codec_case.label()),
17670                "workflow_type": "codec.workflow",
17671                "query_name": "known"
17672            });
17673            codec_case.apply(&mut query);
17674            match serde_json::from_value::<QueryTask>(query) {
17675                Ok(task) => match worker.execute_query_task(task).await {
17676                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17677                    outcome => failures.push(format!(
17678                        "query {} codec returned {outcome:?}",
17679                        codec_case.label()
17680                    )),
17681                },
17682                Err(error) => failures.push(format!(
17683                    "query {} codec failed transport deserialization: {error}",
17684                    codec_case.label()
17685                )),
17686            }
17687        }
17688
17689        assert!(failures.is_empty(), "{}", failures.join("\n"));
17690        assert_eq!(
17691            handler_calls.load(Ordering::SeqCst),
17692            0,
17693            "invalid task codecs must not invoke a handler"
17694        );
17695    }
17696
17697    #[tokio::test]
17698    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17699        for codec_case in [
17700            InvalidTaskPayloadCodec::Missing,
17701            InvalidTaskPayloadCodec::Null,
17702            InvalidTaskPayloadCodec::NonString,
17703        ] {
17704            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17705            let client = Client::builder(server.base_url())
17706                .timeout(Duration::from_secs(2))
17707                .build()
17708                .expect("client");
17709            let mut worker = Worker::new(client, "rust-workers")
17710                .worker_id("codec-worker")
17711                .poll_timeout(Duration::from_millis(10));
17712            let handler_calls = Arc::new(AtomicUsize::new(0));
17713
17714            let calls = Arc::clone(&handler_calls);
17715            worker.register_workflow("codec.workflow", move |_ctx, _args| {
17716                calls.fetch_add(1, Ordering::SeqCst);
17717                async move { Ok(Value::Null) }
17718            });
17719            let calls = Arc::clone(&handler_calls);
17720            worker.register_activity("codec.activity", move |_ctx, _args| {
17721                calls.fetch_add(1, Ordering::SeqCst);
17722                async move { Ok(Value::Null) }
17723            });
17724            let calls = Arc::clone(&handler_calls);
17725            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17726                calls.fetch_add(1, Ordering::SeqCst);
17727                async move { Ok(Value::Null) }
17728            });
17729
17730            assert_eq!(
17731                worker.run_once().await.expect("invalid tasks are settled"),
17732                3,
17733                "all {} codec tasks must be handled",
17734                codec_case.label()
17735            );
17736            assert_eq!(
17737                handler_calls.load(Ordering::SeqCst),
17738                0,
17739                "{} task codecs must fail before every handler",
17740                codec_case.label()
17741            );
17742
17743            for path in [
17744                "/api/worker/workflow-tasks/codec-workflow/fail",
17745                "/api/worker/activity-tasks/codec-activity/fail",
17746                "/api/worker/query-tasks/codec-query/fail",
17747            ] {
17748                let body = server.request_body(path);
17749                assert!(
17750                    body["failure"]["message"]
17751                        .as_str()
17752                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
17753                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
17754                    codec_case.label()
17755                );
17756            }
17757            assert_eq!(
17758                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17759                    ["reason"],
17760                "query_payload_decode_failed"
17761            );
17762            for path in [
17763                "/api/worker/workflow-tasks/codec-workflow/complete",
17764                "/api/worker/activity-tasks/codec-activity/complete",
17765                "/api/worker/query-tasks/codec-query/complete",
17766            ] {
17767                assert_eq!(
17768                    server.request_count(path),
17769                    0,
17770                    "invalid {} codec task reached {path}",
17771                    codec_case.label()
17772                );
17773            }
17774        }
17775    }
17776
17777    #[tokio::test]
17778    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17779        let client = Client::new("http://127.0.0.1:8080").expect("client");
17780        let mut worker = Worker::new(client, "rust-workers");
17781        let handler_calls = Arc::new(AtomicUsize::new(0));
17782
17783        let calls = Arc::clone(&handler_calls);
17784        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17785            calls.fetch_add(1, Ordering::SeqCst);
17786            async move { Ok(Value::Null) }
17787        });
17788        let calls = Arc::clone(&handler_calls);
17789        worker.register_activity("codec.activity", move |_ctx, _args| {
17790            calls.fetch_add(1, Ordering::SeqCst);
17791            async move { Ok(Value::Null) }
17792        });
17793        let calls = Arc::clone(&handler_calls);
17794        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17795            calls.fetch_add(1, Ordering::SeqCst);
17796            async move { Ok(Value::Null) }
17797        });
17798        let calls = Arc::clone(&handler_calls);
17799        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17800            calls.fetch_add(1, Ordering::SeqCst);
17801            async move { Ok(Value::Null) }
17802        });
17803
17804        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17805        workflow.payload_codec = "json".to_string();
17806        workflow.arguments = None;
17807        let error = worker
17808            .execute_workflow_task(workflow)
17809            .expect_err("task codec must be checked before workflow invocation");
17810        assert!(error.to_string().contains("unsupported_payload_codec"));
17811
17812        let activity = ActivityTask {
17813            task_id: "activity-invalid-codec".to_string(),
17814            activity_attempt_id: None,
17815            attempt_id: None,
17816            activity_type: "codec.activity".to_string(),
17817            payload_codec: "unknown".to_string(),
17818            arguments: None,
17819            attempt_number: 1,
17820            lease_owner: None,
17821        };
17822        let error = worker
17823            .execute_activity_task(activity)
17824            .await
17825            .expect_err("task codec must be checked before activity invocation");
17826        assert!(error.to_string().contains("unsupported_payload_codec"));
17827
17828        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17829        update.workflow_update_id = Some("update-invalid-codec".to_string());
17830        update.update_name = Some("known".to_string());
17831        update.history_events.push(history_event(
17832            "UpdateAccepted",
17833            json!({
17834                "update_id": "update-invalid-codec",
17835                "update_name": "known",
17836                "arguments": {"codec": "json", "blob": null}
17837            }),
17838        ));
17839        let error = worker
17840            .execute_workflow_task(update)
17841            .expect_err("nested update codec must be checked before handler lookup");
17842        assert!(error.to_string().contains("unsupported_payload_codec"));
17843
17844        let query: QueryTask = serde_json::from_value(json!({
17845            "query_task_id": "query-invalid-codec",
17846            "workflow_type": "codec.workflow",
17847            "query_name": "known",
17848            "payload_codec": DEFAULT_CODEC,
17849            "workflow_arguments": null,
17850            "query_arguments": null,
17851            "history_export": {
17852                "payloads": {"codec": DEFAULT_CODEC},
17853                "signals": [{
17854                    "name": "empty",
17855                    "payload_codec": "json",
17856                    "arguments": null
17857                }]
17858            }
17859        }))
17860        .expect("query task");
17861        let failure = worker
17862            .execute_query_task(query)
17863            .await
17864            .expect_err("exported signal codec must be checked before query invocation");
17865        assert_eq!(failure.reason, "query_payload_decode_failed");
17866        assert!(failure.message.contains("unsupported_payload_codec"));
17867
17868        let exported_history: QueryTask = serde_json::from_value(json!({
17869            "query_task_id": "query-invalid-history-codec",
17870            "workflow_type": "codec.workflow",
17871            "query_name": "known",
17872            "payload_codec": DEFAULT_CODEC,
17873            "history_export": {
17874                "payloads": {"codec": DEFAULT_CODEC},
17875                "history_events": [{
17876                    "type": "ActivityCompleted",
17877                    "payload": {"payload_codec": "unknown", "result": null}
17878                }]
17879            }
17880        }))
17881        .expect("query task");
17882        let failure = worker
17883            .execute_query_task(exported_history)
17884            .await
17885            .expect_err("exported history codec must be checked before query invocation");
17886        assert_eq!(failure.reason, "query_payload_decode_failed");
17887        assert!(failure.message.contains("unsupported_payload_codec"));
17888        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
17889
17890        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17891        unknown_workflow.arguments = None;
17892        unknown_workflow.history_events.push(history_event(
17893            "SignalReceived",
17894            json!({
17895                "signal_name": "empty",
17896                "payload_codec": "json",
17897                "arguments": null
17898            }),
17899        ));
17900        let error = worker
17901            .execute_workflow_task(unknown_workflow)
17902            .expect_err("history codec must precede unknown workflow outcome");
17903        assert!(error.to_string().contains("unsupported_payload_codec"));
17904
17905        let unknown_activity = ActivityTask {
17906            task_id: "activity-unknown".to_string(),
17907            activity_attempt_id: None,
17908            attempt_id: None,
17909            activity_type: "missing".to_string(),
17910            payload_codec: "json".to_string(),
17911            arguments: None,
17912            attempt_number: 1,
17913            lease_owner: None,
17914        };
17915        let error = worker
17916            .execute_activity_task(unknown_activity)
17917            .await
17918            .expect_err("codec must precede unknown activity outcome");
17919        assert!(error.to_string().contains("unsupported_payload_codec"));
17920
17921        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17922        unknown_update.payload_codec = "json".to_string();
17923        unknown_update.arguments = None;
17924        unknown_update.workflow_update_id = Some("update-unknown".to_string());
17925        unknown_update.update_name = Some("missing".to_string());
17926        let error = worker
17927            .execute_workflow_task(unknown_update)
17928            .expect_err("codec must precede fail_update shortcut");
17929        assert!(error.to_string().contains("unsupported_payload_codec"));
17930
17931        let unknown_query: QueryTask = serde_json::from_value(json!({
17932            "query_task_id": "query-unknown",
17933            "workflow_type": "missing",
17934            "query_name": "missing",
17935            "payload_codec": "json",
17936            "workflow_arguments": null,
17937            "query_arguments": null
17938        }))
17939        .expect("query task");
17940        let failure = worker
17941            .execute_query_task(unknown_query)
17942            .await
17943            .expect_err("codec must precede unknown query outcome");
17944        assert_eq!(failure.reason, "query_payload_decode_failed");
17945        assert!(failure.message.contains("unsupported_payload_codec"));
17946    }
17947
17948    #[tokio::test]
17949    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
17950        let client = Client::new("http://127.0.0.1:8080").expect("client");
17951        let worker = Worker::new(client, "rust-workers");
17952
17953        for event_type in ["SignalReceived", "SignalApplied"] {
17954            for (payload_field, codec) in [
17955                ("value", "json"),
17956                ("input", "unknown"),
17957                ("arguments", "json"),
17958            ] {
17959                let payload = json!({
17960                    "signal_name": "empty",
17961                    payload_field: {"codec": codec, "blob": null}
17962                });
17963                let workflow = workflow_task(
17964                    "missing",
17965                    vec![history_event(event_type, payload.clone())],
17966                    DEFAULT_CODEC,
17967                );
17968                let error = worker
17969                    .execute_workflow_task(workflow)
17970                    .expect_err("signal payload codec must precede unknown workflow outcome");
17971                assert!(
17972                    error.to_string().contains("unsupported_payload_codec"),
17973                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
17974                );
17975
17976                let query: QueryTask = serde_json::from_value(json!({
17977                    "query_task_id": format!("query-{event_type}-{payload_field}"),
17978                    "workflow_type": "missing",
17979                    "query_name": "missing",
17980                    "payload_codec": DEFAULT_CODEC,
17981                    "workflow_arguments": null,
17982                    "query_arguments": null,
17983                    "history_events": [{
17984                        "event_type": event_type,
17985                        "payload": payload
17986                    }]
17987                }))
17988                .expect("query task");
17989                let failure = worker
17990                    .execute_query_task(query)
17991                    .await
17992                    .expect_err("signal payload codec must precede unknown query outcome");
17993                assert_eq!(
17994                    failure.reason, "query_payload_decode_failed",
17995                    "{event_type}.{payload_field} returned an unrelated query outcome"
17996                );
17997                assert!(
17998                    failure.message.contains("unsupported_payload_codec"),
17999                    "{event_type}.{payload_field} returned an unrelated query error: {}",
18000                    failure.message
18001                );
18002            }
18003        }
18004    }
18005
18006    #[test]
18007    fn workflow_context_schedules_activity_until_completion_is_in_history() {
18008        let ctx = WorkflowContext {
18009            state: Arc::new(Mutex::new(
18010                WorkflowState::new_with_identity(
18011                    Vec::new(),
18012                    Some("wf-parent".to_string()),
18013                    Some("run-parent".to_string()),
18014                    "rust-workers".to_string(),
18015                    DEFAULT_CODEC.to_string(),
18016                    None,
18017                )
18018                .expect("workflow state"),
18019            )),
18020        };
18021
18022        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18023        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18024        assert!(matches!(
18025            call.as_mut().poll(&mut task_context),
18026            Poll::Pending
18027        ));
18028
18029        let commands = ctx.take_commands().expect("commands");
18030        assert_eq!(commands[0]["type"], "schedule_activity");
18031        assert_eq!(commands[0]["activity_type"], "hello.activity");
18032    }
18033
18034    #[test]
18035    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18036        let ctx = workflow_context(Vec::new());
18037        let options = ActivityOptions::new()
18038            .task_queue("payments")
18039            .retry_policy(
18040                ActivityRetryPolicy::new(4)
18041                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18042                    .non_retryable_error_type("ValidationError"),
18043            )
18044            .start_to_close_timeout(Duration::from_secs(120))
18045            .schedule_to_start_timeout(Duration::from_secs(10))
18046            .schedule_to_close_timeout(Duration::from_secs(300))
18047            .heartbeat_timeout(Duration::from_secs(15));
18048        let mut call = Box::pin(ctx.activity_with_options(
18049            "charge-card",
18050            options,
18051            json!([{"order_id": "o-1"}]),
18052        ));
18053        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18054
18055        assert!(matches!(
18056            call.as_mut().poll(&mut task_context),
18057            Poll::Pending
18058        ));
18059        assert!(matches!(
18060            call.as_mut().poll(&mut task_context),
18061            Poll::Pending
18062        ));
18063
18064        let commands = ctx.take_commands().expect("activity command");
18065        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18066        assert_eq!(commands[0]["queue"], "payments");
18067        assert_eq!(
18068            commands[0]["retry_policy"],
18069            json!({
18070                "max_attempts": 4,
18071                "backoff_seconds": [1, 3, 9],
18072                "non_retryable_error_types": ["ValidationError"],
18073            })
18074        );
18075        assert_eq!(commands[0]["start_to_close_timeout"], 120);
18076        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18077        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18078        assert_eq!(commands[0]["heartbeat_timeout"], 15);
18079    }
18080
18081    #[test]
18082    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18083        let ctx = workflow_context(Vec::new());
18084        let options = ActivityOptions::new().retry_policy(
18085            ActivityRetryPolicy::new(3)
18086                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18087        );
18088        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18089        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18090
18091        assert!(matches!(
18092            call.as_mut().poll(&mut task_context),
18093            Poll::Pending
18094        ));
18095        assert_eq!(
18096            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18097            json!([1, 2])
18098        );
18099    }
18100
18101    #[test]
18102    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18103        let cases = [
18104            (
18105                ActivityOptions::new().task_queue("  "),
18106                ActivityOptionsErrorKind::EmptyTaskQueue,
18107            ),
18108            (
18109                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18110                ActivityOptionsErrorKind::EmptyRetryPolicy,
18111            ),
18112            (
18113                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18114                ActivityOptionsErrorKind::InvalidMaxAttempts,
18115            ),
18116            (
18117                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18118                    max_attempts: None,
18119                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18120                    non_retryable_error_types: Vec::new(),
18121                }),
18122                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18123            ),
18124            (
18125                ActivityOptions::new().retry_policy(
18126                    ActivityRetryPolicy::new(2)
18127                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18128                ),
18129                ActivityOptionsErrorKind::TooManyBackoffIntervals,
18130            ),
18131            (
18132                ActivityOptions::new().retry_policy(
18133                    ActivityRetryPolicy::new(2).exponential_backoff(
18134                        Duration::from_secs(1),
18135                        0,
18136                        None,
18137                    ),
18138                ),
18139                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18140            ),
18141            (
18142                ActivityOptions::new()
18143                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
18144                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18145            ),
18146            (
18147                ActivityOptions::new().retry_policy(
18148                    ActivityRetryPolicy::new(10_002).exponential_backoff(
18149                        Duration::from_secs(1),
18150                        1,
18151                        None,
18152                    ),
18153                ),
18154                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18155            ),
18156            (
18157                ActivityOptions::new().retry_policy(
18158                    ActivityRetryPolicy::new(2)
18159                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18160                ),
18161                ActivityOptionsErrorKind::BackoffOverflow,
18162            ),
18163        ];
18164
18165        for (options, expected_kind) in cases {
18166            let ctx = workflow_context(Vec::new());
18167            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18168            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18169            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18170                call.as_mut().poll(&mut task_context)
18171            else {
18172                panic!("expected typed activity validation error");
18173            };
18174            assert_eq!(error.kind, expected_kind);
18175            assert!(ctx.take_commands().expect("commands").is_empty());
18176        }
18177    }
18178
18179    #[test]
18180    fn activity_options_validate_positive_and_ordered_timeouts() {
18181        let zero_timeout_cases = [
18182            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18183            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18184            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18185            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18186        ];
18187        for options in zero_timeout_cases {
18188            assert_eq!(
18189                options.validate().expect_err("zero timeout").kind,
18190                ActivityOptionsErrorKind::TimeoutNotPositive
18191            );
18192        }
18193
18194        let ordering_cases = [
18195            ActivityOptions::new()
18196                .heartbeat_timeout(Duration::from_secs(11))
18197                .start_to_close_timeout(Duration::from_secs(10)),
18198            ActivityOptions::new()
18199                .start_to_close_timeout(Duration::from_secs(31))
18200                .schedule_to_close_timeout(Duration::from_secs(30)),
18201            ActivityOptions::new()
18202                .schedule_to_start_timeout(Duration::from_secs(31))
18203                .schedule_to_close_timeout(Duration::from_secs(30)),
18204        ];
18205        for options in ordering_cases {
18206            assert_eq!(
18207                options.validate().expect_err("timeout order").kind,
18208                ActivityOptionsErrorKind::TimeoutOrder
18209            );
18210        }
18211
18212        assert_eq!(
18213            ActivityOptions::new()
18214                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18215                .validate()
18216                .expect_err("protocol integer overflow")
18217                .kind,
18218            ActivityOptionsErrorKind::TimeoutOverflow
18219        );
18220    }
18221
18222    #[test]
18223    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18224        let ctx = workflow_context(completed_retry_activity_history());
18225        let mut call =
18226            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18227        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18228
18229        assert!(matches!(
18230            call.as_mut().poll(&mut task_context),
18231            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18232        ));
18233        assert!(ctx.take_commands().expect("commands").is_empty());
18234        ctx.ensure_history_consumed().expect("history consumed");
18235    }
18236
18237    #[test]
18238    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18239        let mut options = retry_activity_options();
18240        options
18241            .retry_policy
18242            .as_mut()
18243            .expect("retry policy")
18244            .non_retryable_error_types
18245            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18246
18247        let new_ctx = workflow_context(Vec::new());
18248        let mut new_call =
18249            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18250        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18251        assert!(matches!(
18252            new_call.as_mut().poll(&mut task_context),
18253            Poll::Pending
18254        ));
18255        let commands = new_ctx.take_commands().expect("commands");
18256        assert_eq!(commands.len(), 1);
18257        assert_eq!(
18258            commands[0]["retry_policy"]["non_retryable_error_types"],
18259            json!(["PermanentError"])
18260        );
18261
18262        let replay_ctx = workflow_context(completed_retry_activity_history());
18263        let mut replay_call =
18264            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18265        assert!(matches!(
18266            replay_call.as_mut().poll(&mut task_context),
18267            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18268        ));
18269        assert!(replay_ctx.take_commands().expect("commands").is_empty());
18270        replay_ctx
18271            .ensure_history_consumed()
18272            .expect("history consumed");
18273    }
18274
18275    #[test]
18276    fn replayed_intermediate_retry_remains_pending_across_restarts() {
18277        let history = completed_retry_activity_history()
18278            .into_iter()
18279            .take(3)
18280            .collect::<Vec<_>>();
18281
18282        for _restart in 0..2 {
18283            let ctx = workflow_context(history.clone());
18284            let mut call =
18285                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18286            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18287            assert!(matches!(
18288                call.as_mut().poll(&mut task_context),
18289                Poll::Pending
18290            ));
18291            assert!(ctx.take_commands().expect("commands").is_empty());
18292        }
18293    }
18294
18295    #[test]
18296    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18297        let mut changed_queue = retry_activity_options();
18298        changed_queue.task_queue = Some("different-queue".to_string());
18299
18300        let mut changed_max_attempts = retry_activity_options();
18301        let retry_policy = changed_max_attempts
18302            .retry_policy
18303            .as_mut()
18304            .expect("retry policy");
18305        retry_policy.max_attempts = Some(4);
18306
18307        let mut changed_backoff = retry_activity_options();
18308        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18309        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18310            Duration::from_secs(3),
18311            Duration::from_secs(4),
18312        ]));
18313
18314        let mut changed_non_retryable_types = retry_activity_options();
18315        let retry_policy = changed_non_retryable_types
18316            .retry_policy
18317            .as_mut()
18318            .expect("retry policy");
18319        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18320
18321        let mut changed_start_to_close = retry_activity_options();
18322        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18323        let mut changed_schedule_to_start = retry_activity_options();
18324        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18325        let mut changed_schedule_to_close = retry_activity_options();
18326        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18327        let mut changed_heartbeat = retry_activity_options();
18328        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18329
18330        let cases = [
18331            (changed_queue, "activity_task_queue_mismatch"),
18332            (changed_max_attempts, "activity_retry_policy_mismatch"),
18333            (changed_backoff, "activity_retry_policy_mismatch"),
18334            (
18335                changed_non_retryable_types,
18336                "activity_retry_policy_mismatch",
18337            ),
18338            (changed_start_to_close, "activity_retry_policy_mismatch"),
18339            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18340            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18341            (changed_heartbeat, "activity_retry_policy_mismatch"),
18342        ];
18343
18344        for (options, expected_reason) in cases {
18345            let ctx = workflow_context(completed_retry_activity_history());
18346            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18347            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18348            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18349                call.as_mut().poll(&mut task_context)
18350            else {
18351                panic!("changed activity options must fail replay");
18352            };
18353            assert_eq!(failure.reason, expected_reason);
18354            assert_eq!(failure.sequence, Some(1));
18355            assert!(ctx.take_commands().expect("commands").is_empty());
18356        }
18357    }
18358
18359    #[test]
18360    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18361        let cases = [
18362            (
18363                "execution_mode",
18364                json!("local"),
18365                "activity_execution_mode_mismatch",
18366            ),
18367            (
18368                "snapshot_version",
18369                json!(2),
18370                "activity_retry_policy_mismatch",
18371            ),
18372        ];
18373
18374        for (field, value, expected_reason) in cases {
18375            let mut history = completed_retry_activity_history();
18376            let activity = history[0].payload["activity"]
18377                .as_object_mut()
18378                .expect("activity snapshot");
18379            if field == "execution_mode" {
18380                activity.insert(field.to_string(), value);
18381            } else {
18382                activity["retry_policy"]
18383                    .as_object_mut()
18384                    .expect("retry snapshot")
18385                    .insert(field.to_string(), value);
18386            }
18387
18388            let ctx = workflow_context(history);
18389            let mut call =
18390                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18391            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18392            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18393                call.as_mut().poll(&mut task_context)
18394            else {
18395                panic!("changed {field} must fail replay");
18396            };
18397            assert_eq!(failure.reason, expected_reason);
18398            assert_eq!(failure.sequence, Some(1));
18399            assert!(ctx.take_commands().expect("commands").is_empty());
18400        }
18401    }
18402
18403    #[test]
18404    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18405        let mut history = completed_retry_activity_history();
18406        let activity = history[0].payload["activity"]
18407            .as_object_mut()
18408            .expect("activity snapshot");
18409        activity.remove("execution_mode");
18410        activity.remove("retry_policy");
18411
18412        let mut current = retry_activity_options();
18413        current.start_to_close_timeout = Some(Duration::from_secs(45));
18414        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18415        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18416        current.heartbeat_timeout = Some(Duration::from_secs(12));
18417
18418        let ctx = workflow_context(history);
18419        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18420        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18421        assert!(matches!(
18422            call.as_mut().poll(&mut task_context),
18423            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18424        ));
18425        assert!(ctx.take_commands().expect("commands").is_empty());
18426        ctx.ensure_history_consumed().expect("history consumed");
18427    }
18428
18429    #[test]
18430    fn terminal_activity_failed_after_start_returns_typed_failure() {
18431        let history = vec![
18432            history_event(
18433                "ActivityScheduled",
18434                json!({
18435                    "sequence": 1,
18436                    "activity_type": "flaky",
18437                    "activity_execution_id": "act-terminal",
18438                    "activity": {
18439                        "id": "act-terminal",
18440                        "sequence": 1,
18441                        "type": "flaky",
18442                        "queue": "critical-activities",
18443                        "retry_policy": {
18444                            "snapshot_version": 1,
18445                            "max_attempts": 3,
18446                            "backoff_seconds": [2, 4],
18447                            "non_retryable_error_types": ["PermanentError"]
18448                        }
18449                    }
18450                }),
18451            ),
18452            history_event(
18453                "ActivityStarted",
18454                json!({
18455                    "sequence": 1,
18456                    "activity_type": "flaky",
18457                    "activity_execution_id": "act-terminal",
18458                    "activity_attempt_id": "attempt-1",
18459                    "attempt_number": 1
18460                }),
18461            ),
18462            history_event(
18463                "ActivityFailed",
18464                json!({
18465                    "sequence": 1,
18466                    "activity_type": "flaky",
18467                    "activity_execution_id": "act-terminal",
18468                    "activity_attempt_id": "attempt-1",
18469                    "attempt_number": 1,
18470                    "failure_id": "failure-terminal",
18471                    "failure_category": "activity",
18472                    "exception_type": "PermanentError",
18473                    "message": "cannot retry",
18474                    "non_retryable": true
18475                }),
18476            ),
18477        ];
18478        let ctx = workflow_context(history);
18479        let mut call =
18480            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18481        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18482
18483        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18484            call.as_mut().poll(&mut task_context)
18485        else {
18486            panic!("terminal ActivityFailed must settle the activity future");
18487        };
18488        assert_eq!(failure.kind, ActivityFailureKind::Failed);
18489        assert_eq!(
18490            failure.activity_execution_id.as_deref(),
18491            Some("act-terminal")
18492        );
18493        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18494        assert!(failure.non_retryable);
18495        assert!(ctx.take_commands().expect("commands").is_empty());
18496        ctx.ensure_history_consumed().expect("history consumed");
18497    }
18498
18499    #[test]
18500    fn activity_terminal_events_return_machine_readable_failures() {
18501        let cases = [
18502            (
18503                "ActivityFailed",
18504                json!({
18505                    "sequence": 1,
18506                    "activity_type": "charge-card",
18507                    "activity_execution_id": "act-1",
18508                    "activity_attempt_id": "attempt-2",
18509                    "attempt_number": 2,
18510                    "failure_id": "failure-1",
18511                    "failure_category": "activity",
18512                    "exception_type": "PaymentDeclined",
18513                    "exception_class": "payments.PaymentDeclined",
18514                    "message": "card declined",
18515                    "non_retryable": true
18516                }),
18517                ActivityFailureKind::Failed,
18518                "activity",
18519            ),
18520            (
18521                "ActivityCancelled",
18522                json!({
18523                    "sequence": 1,
18524                    "activity_type": "charge-card",
18525                    "activity_execution_id": "act-1",
18526                    "activity_attempt_id": "attempt-1"
18527                }),
18528                ActivityFailureKind::Cancelled,
18529                "cancelled",
18530            ),
18531        ];
18532
18533        for (event_type, payload, expected_kind, expected_reason) in cases {
18534            let ctx = workflow_context(vec![history_event(event_type, payload)]);
18535            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18536            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18537            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18538                call.as_mut().poll(&mut task_context)
18539            else {
18540                panic!("expected terminal activity failure");
18541            };
18542            assert_eq!(failure.kind, expected_kind);
18543            assert_eq!(failure.reason, expected_reason);
18544            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18545            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18546        }
18547    }
18548
18549    #[test]
18550    fn every_activity_timeout_class_is_typed() {
18551        for timeout_kind in [
18552            "start_to_close",
18553            "schedule_to_start",
18554            "schedule_to_close",
18555            "heartbeat",
18556        ] {
18557            let ctx = workflow_context(vec![history_event(
18558                "ActivityTimedOut",
18559                json!({
18560                    "sequence": 1,
18561                    "activity_type": "slow",
18562                    "activity_execution_id": "act-timeout",
18563                    "activity_attempt_id": "attempt-timeout",
18564                    "failure_category": "timeout",
18565                    "timeout_kind": timeout_kind,
18566                    "message": "deadline expired"
18567                }),
18568            )]);
18569            let mut call = Box::pin(ctx.activity("slow", json!([])));
18570            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18571            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18572                call.as_mut().poll(&mut task_context)
18573            else {
18574                panic!("expected timeout failure");
18575            };
18576            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18577            assert_eq!(failure.reason, timeout_kind);
18578            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18579            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18580        }
18581    }
18582
18583    #[test]
18584    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18585        let ctx = workflow_context(Vec::new());
18586        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18587        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18588
18589        assert!(matches!(
18590            sleep.as_mut().poll(&mut task_context),
18591            Poll::Pending
18592        ));
18593        assert!(matches!(
18594            sleep.as_mut().poll(&mut task_context),
18595            Poll::Pending
18596        ));
18597
18598        let commands = ctx.take_commands().expect("timer command");
18599        assert_eq!(
18600            commands,
18601            vec![json!({
18602                "type": "start_timer",
18603                "delay_seconds": 2,
18604            })]
18605        );
18606    }
18607
18608    #[test]
18609    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18610        let history = vec![
18611            history_event(
18612                "TimerScheduled",
18613                json!({
18614                    "sequence": 1,
18615                    "timer_id": "timer-1",
18616                    "delay_seconds": 5,
18617                    "fire_at": "2026-07-11T12:00:05Z",
18618                }),
18619            ),
18620            history_event(
18621                "TimerFired",
18622                json!({
18623                    "sequence": 1,
18624                    "timer_id": "timer-1",
18625                    "delay_seconds": 5,
18626                    "fire_at": "2026-07-11T12:00:05Z",
18627                    "fired_at": "2026-07-11T12:00:05Z",
18628                }),
18629            ),
18630        ];
18631
18632        for _restart in 0..2 {
18633            let ctx = workflow_context(history.clone());
18634            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18635            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18636            assert!(matches!(
18637                sleep.as_mut().poll(&mut task_context),
18638                Poll::Ready(Ok(()))
18639            ));
18640            assert!(ctx.take_commands().expect("commands").is_empty());
18641            ctx.ensure_history_consumed().expect("history consumed");
18642        }
18643    }
18644
18645    #[test]
18646    fn workflow_sleep_rejects_changed_delay_during_replay() {
18647        let ctx = workflow_context(vec![
18648            history_event(
18649                "TimerScheduled",
18650                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18651            ),
18652            history_event(
18653                "TimerFired",
18654                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18655            ),
18656        ]);
18657        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18658        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18659
18660        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18661            sleep.as_mut().poll(&mut task_context)
18662        else {
18663            panic!("changed timer delay must be rejected");
18664        };
18665        assert_eq!(failure.reason, "timer_delay_mismatch");
18666        assert_eq!(failure.sequence, Some(1));
18667    }
18668
18669    #[test]
18670    fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18671        let ctx = workflow_context(Vec::new());
18672        let mut wait = Box::pin(
18673            ctx.wait_condition(
18674                ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18675                    .timeout(Duration::from_millis(60_001)),
18676                || Ok(false),
18677            ),
18678        );
18679        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18680
18681        assert!(matches!(
18682            wait.as_mut().poll(&mut task_context),
18683            Poll::Pending
18684        ));
18685        assert!(matches!(
18686            wait.as_mut().poll(&mut task_context),
18687            Poll::Pending
18688        ));
18689        assert_eq!(
18690            ctx.take_commands().expect("condition command"),
18691            vec![json!({
18692                "type": "open_condition_wait",
18693                "condition_wait_occurrence_id": "rust:condition-wait:0",
18694                "condition_key": "approval.ready",
18695                "condition_definition_fingerprint": "sha256:approval-v1",
18696                "timeout_seconds": 61,
18697            })]
18698        );
18699    }
18700
18701    #[test]
18702    fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18703        let ctx = workflow_context(Vec::new());
18704        let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18705        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18706        assert!(matches!(
18707            satisfied.as_mut().poll(&mut task_context),
18708            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18709        ));
18710
18711        let mut timed_out = Box::pin(wait_condition!(
18712            ctx,
18713            "no-wait",
18714            timeout: Duration::ZERO,
18715            || Ok(false),
18716        ));
18717        assert!(matches!(
18718            timed_out.as_mut().poll(&mut task_context),
18719            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18720        ));
18721        assert!(ctx.take_commands().expect("commands").is_empty());
18722    }
18723
18724    #[test]
18725    fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18726        let signal_history = vec![
18727            history_event(
18728                "ConditionWaitOpened",
18729                json!({
18730                    "sequence": 4,
18731                    "condition_wait_id": "condition:4",
18732                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18733                    "condition_key": "approval",
18734                    "condition_definition_fingerprint": "sha256:approval-v1",
18735                    "timeout_seconds": 30,
18736                }),
18737            ),
18738            history_event(
18739                "SignalReceived",
18740                json!({
18741                    "workflow_sequence": 4,
18742                    "signal_name": "approve",
18743                    "arguments": fixture_envelope(json!(["Ada"])),
18744                }),
18745            ),
18746        ];
18747        for _worker_before_or_after_restart in 0..2 {
18748            let ctx = workflow_context(signal_history.clone());
18749            let predicate_ctx = ctx.clone();
18750            let mut wait = Box::pin(
18751                ctx.wait_condition(
18752                    ConditionWaitOptions::new("approval", "sha256:approval-v1")
18753                        .timeout(Duration::from_secs(30)),
18754                    move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18755                ),
18756            );
18757            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18758            assert!(matches!(
18759                wait.as_mut().poll(&mut task_context),
18760                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18761            ));
18762            assert!(ctx.take_commands().expect("commands").is_empty());
18763            ctx.ensure_history_consumed().expect("condition consumed");
18764        }
18765
18766        let update_history = vec![
18767            history_event(
18768                "ConditionWaitOpened",
18769                json!({
18770                    "sequence": 7,
18771                    "condition_wait_id": "condition:7",
18772                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18773                    "condition_key": "update-approval",
18774                    "condition_definition_fingerprint": "sha256:update-approval-v1",
18775                }),
18776            ),
18777            history_event(
18778                "UpdateApplied",
18779                json!({
18780                    "sequence": 7,
18781                    "update_id": "update-1",
18782                    "update_name": "approve",
18783                    "arguments": fixture_envelope(json!([true])),
18784                }),
18785            ),
18786        ];
18787        let ctx = workflow_context(update_history);
18788        let predicate_ctx = ctx.clone();
18789        let mut wait = Box::pin(ctx.wait_condition(
18790            ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18791            move || {
18792                Ok(predicate_ctx
18793                    .updates("approve")?
18794                    .first()
18795                    .and_then(|arguments| arguments.first())
18796                    .and_then(Value::as_bool)
18797                    == Some(true))
18798            },
18799        ));
18800        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18801        assert!(matches!(
18802            wait.as_mut().poll(&mut task_context),
18803            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18804        ));
18805        assert!(ctx.take_commands().expect("commands").is_empty());
18806        ctx.ensure_history_consumed().expect("condition consumed");
18807    }
18808
18809    #[test]
18810    fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18811        let open_history = vec![
18812            history_event(
18813                "ConditionWaitOpened",
18814                json!({
18815                    "sequence": 3,
18816                    "condition_wait_id": "condition:3",
18817                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18818                    "condition_key": "two-votes",
18819                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18820                    "timeout_seconds": 120,
18821                }),
18822            ),
18823            history_event(
18824                "SignalReceived",
18825                json!({
18826                    "workflow_sequence": 3,
18827                    "signal_name": "vote",
18828                    "arguments": fixture_envelope(json!(["first"])),
18829                }),
18830            ),
18831        ];
18832        for _worker_before_or_after_restart in 0..2 {
18833            let ctx = workflow_context(open_history.clone());
18834            let predicate_ctx = ctx.clone();
18835            let mut wait = Box::pin(
18836                ctx.wait_condition(
18837                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18838                        .timeout(Duration::from_secs(120)),
18839                    move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18840                ),
18841            );
18842            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18843            assert!(matches!(
18844                wait.as_mut().poll(&mut task_context),
18845                Poll::Pending
18846            ));
18847            assert_eq!(
18848                ctx.take_commands().expect("reopened condition"),
18849                vec![json!({
18850                    "type": "open_condition_wait",
18851                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18852                    "condition_key": "two-votes",
18853                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18854                    "timeout_seconds": 120,
18855                })]
18856            );
18857        }
18858
18859        let satisfied_ctx = workflow_context(vec![
18860            history_event(
18861                "ConditionWaitOpened",
18862                json!({
18863                    "sequence": 5,
18864                    "condition_wait_id": "condition:5",
18865                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18866                    "condition_key": "approval",
18867                    "condition_definition_fingerprint": "sha256:approval-v1",
18868                }),
18869            ),
18870            history_event(
18871                "ConditionWaitSatisfied",
18872                json!({
18873                    "sequence": 5,
18874                    "condition_wait_id": "condition:5",
18875                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18876                    "condition_key": "approval",
18877                    "condition_definition_fingerprint": "sha256:approval-v1",
18878                }),
18879            ),
18880        ]);
18881        let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18882            ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18883            || Ok(false),
18884        ));
18885        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18886        assert!(matches!(
18887            satisfied.as_mut().poll(&mut task_context),
18888            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18889        ));
18890
18891        let timed_out_ctx = workflow_context(vec![
18892            history_event(
18893                "ConditionWaitOpened",
18894                json!({
18895                    "sequence": 8,
18896                    "condition_wait_id": "condition:8",
18897                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18898                    "condition_key": "approval-timeout",
18899                    "condition_definition_fingerprint": "sha256:approval-timeout-v1",
18900                    "timeout_seconds": 5,
18901                }),
18902            ),
18903            history_event(
18904                "TimerScheduled",
18905                json!({
18906                    "sequence": 9,
18907                    "timer_id": "condition-timer:9",
18908                    "timer_kind": "condition_timeout",
18909                    "condition_wait_id": "condition:8",
18910                    "delay_seconds": 5,
18911                }),
18912            ),
18913            history_event(
18914                "TimerFired",
18915                json!({
18916                    "sequence": 9,
18917                    "timer_id": "condition-timer:9",
18918                    "timer_kind": "condition_timeout",
18919                    "condition_wait_id": "condition:8",
18920                    "delay_seconds": 5,
18921                }),
18922            ),
18923        ]);
18924        let mut timed_out = Box::pin(
18925            timed_out_ctx.wait_condition(
18926                ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
18927                    .timeout(Duration::from_secs(5)),
18928                || Ok(true),
18929            ),
18930        );
18931        assert!(matches!(
18932            timed_out.as_mut().poll(&mut task_context),
18933            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18934        ));
18935    }
18936
18937    #[test]
18938    fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
18939        let history = vec![
18940            history_event(
18941                "ConditionWaitOpened",
18942                json!({
18943                    "sequence": 3,
18944                    "condition_wait_id": "condition:3",
18945                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18946                    "condition_key": "two-votes",
18947                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18948                }),
18949            ),
18950            history_event(
18951                "SignalReceived",
18952                json!({
18953                    "workflow_sequence": 3,
18954                    "signal_name": "vote",
18955                    "arguments": fixture_envelope(json!(["first"])),
18956                }),
18957            ),
18958            history_event(
18959                "ConditionWaitSatisfied",
18960                json!({
18961                    "sequence": 3,
18962                    "condition_wait_id": "condition:3",
18963                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18964                    "condition_key": "two-votes",
18965                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18966                }),
18967            ),
18968            history_event(
18969                "ConditionWaitOpened",
18970                json!({
18971                    "sequence": 5,
18972                    "condition_wait_id": "condition:5",
18973                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18974                    "condition_key": "two-votes",
18975                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18976                }),
18977            ),
18978            history_event(
18979                "SignalReceived",
18980                json!({
18981                    "workflow_sequence": 5,
18982                    "signal_name": "vote",
18983                    "arguments": fixture_envelope(json!(["second"])),
18984                }),
18985            ),
18986            history_event(
18987                "ConditionWaitSatisfied",
18988                json!({
18989                    "sequence": 5,
18990                    "condition_wait_id": "condition:5",
18991                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18992                    "condition_key": "two-votes",
18993                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18994                }),
18995            ),
18996        ];
18997        for _cold_worker_or_restart in 0..2 {
18998            let ctx = workflow_context(history.clone());
18999            let predicate_ctx = ctx.clone();
19000            let mut wait = Box::pin(ctx.wait_condition(
19001                ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19002                move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19003            ));
19004            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19005
19006            assert!(matches!(
19007                wait.as_mut().poll(&mut task_context),
19008                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19009            ));
19010            assert!(ctx.take_commands().expect("commands").is_empty());
19011            ctx.ensure_history_consumed()
19012                .expect("every physical wait-open is consumed");
19013        }
19014    }
19015
19016    #[test]
19017    fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19018        let history = vec![
19019            history_event(
19020                "ConditionWaitOpened",
19021                json!({
19022                    "sequence": 3,
19023                    "condition_wait_id": "condition:3",
19024                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19025                    "condition_key": "approved",
19026                    "condition_definition_fingerprint": "sha256:approved-v1",
19027                }),
19028            ),
19029            history_event(
19030                "UpdateApplied",
19031                json!({
19032                    "sequence": 3,
19033                    "update_id": "update-1",
19034                    "update_name": "approve",
19035                    "arguments": fixture_envelope(json!([false])),
19036                }),
19037            ),
19038            history_event(
19039                "ConditionWaitOpened",
19040                json!({
19041                    "sequence": 5,
19042                    "condition_wait_id": "condition:5",
19043                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19044                    "condition_key": "approved",
19045                    "condition_definition_fingerprint": "sha256:approved-v1",
19046                }),
19047            ),
19048            history_event(
19049                "UpdateApplied",
19050                json!({
19051                    "sequence": 5,
19052                    "update_id": "update-2",
19053                    "update_name": "approve",
19054                    "arguments": fixture_envelope(json!([true])),
19055                }),
19056            ),
19057        ];
19058
19059        for _cold_worker_or_restart in 0..2 {
19060            let ctx = workflow_context(history.clone());
19061            let predicate_ctx = ctx.clone();
19062            let mut wait = Box::pin(ctx.wait_condition(
19063                ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19064                move || {
19065                    Ok(predicate_ctx
19066                        .updates("approve")?
19067                        .last()
19068                        .and_then(|arguments| arguments.first())
19069                        .and_then(Value::as_bool)
19070                        == Some(true))
19071                },
19072            ));
19073            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19074
19075            assert!(matches!(
19076                wait.as_mut().poll(&mut task_context),
19077                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19078            ));
19079            assert!(ctx.take_commands().expect("commands").is_empty());
19080            ctx.ensure_history_consumed()
19081                .expect("every update-driven reopen is consumed");
19082        }
19083    }
19084
19085    #[test]
19086    fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19087        for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19088            ("shared", "sha256:first", "shared", "sha256:second"),
19089            ("first", "sha256:shared", "second", "sha256:shared"),
19090            ("shared", "sha256:shared", "shared", "sha256:shared"),
19091            ("first", "sha256:first", "second", "sha256:second"),
19092        ] {
19093            let history = vec![
19094                history_event(
19095                    "ConditionWaitOpened",
19096                    json!({
19097                        "sequence": 3,
19098                        "condition_wait_id": "condition:3",
19099                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19100                        "condition_key": first_key,
19101                        "condition_definition_fingerprint": first_fingerprint,
19102                    }),
19103                ),
19104                history_event(
19105                    "ConditionWaitSatisfied",
19106                    json!({
19107                        "sequence": 3,
19108                        "condition_wait_id": "condition:3",
19109                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19110                        "condition_key": first_key,
19111                        "condition_definition_fingerprint": first_fingerprint,
19112                    }),
19113                ),
19114                history_event(
19115                    "ConditionWaitOpened",
19116                    json!({
19117                        "sequence": 4,
19118                        "condition_wait_id": "condition:4",
19119                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19120                        "condition_key": second_key,
19121                        "condition_definition_fingerprint": second_fingerprint,
19122                    }),
19123                ),
19124                history_event(
19125                    "ConditionWaitSatisfied",
19126                    json!({
19127                        "sequence": 4,
19128                        "condition_wait_id": "condition:4",
19129                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19130                        "condition_key": second_key,
19131                        "condition_definition_fingerprint": second_fingerprint,
19132                    }),
19133                ),
19134            ];
19135            for _cold_worker_or_restart in 0..2 {
19136                let ctx = workflow_context(history.clone());
19137                let mut task_context = TaskContext::from_waker(noop_waker_ref());
19138                let mut first = Box::pin(ctx.wait_condition(
19139                    ConditionWaitOptions::new(first_key, first_fingerprint),
19140                    || Ok(false),
19141                ));
19142                assert!(matches!(
19143                    first.as_mut().poll(&mut task_context),
19144                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19145                ));
19146
19147                let mut second = Box::pin(ctx.wait_condition(
19148                    ConditionWaitOptions::new(second_key, second_fingerprint),
19149                    || Ok(false),
19150                ));
19151                assert!(matches!(
19152                    second.as_mut().poll(&mut task_context),
19153                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19154                ));
19155                assert!(ctx.take_commands().expect("commands").is_empty());
19156                ctx.ensure_history_consumed()
19157                    .expect("each authored wait consumes one occurrence");
19158            }
19159        }
19160    }
19161
19162    #[test]
19163    fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19164        fn worker() -> Worker {
19165            let client = Client::new("http://127.0.0.1:8080").expect("client");
19166            let mut worker = Worker::new(client, "rust-workers");
19167            worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19168                let mut outcomes = Vec::new();
19169                for _ in 0..2 {
19170                    outcomes.push(
19171                        ctx.wait_condition(
19172                            ConditionWaitOptions::new("shared", "sha256:shared"),
19173                            || Ok(false),
19174                        )
19175                        .await?,
19176                    );
19177                }
19178                Ok(json!(outcomes))
19179            });
19180            worker
19181        }
19182
19183        let task = workflow_task(
19184            "rust.condition-loop",
19185            vec![
19186                history_event(
19187                    "ConditionWaitOpened",
19188                    json!({
19189                        "sequence": 1,
19190                        "condition_wait_id": "condition:1",
19191                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19192                        "condition_key": "shared",
19193                        "condition_definition_fingerprint": "sha256:shared",
19194                    }),
19195                ),
19196                history_event(
19197                    "ConditionWaitSatisfied",
19198                    json!({
19199                        "sequence": 1,
19200                        "condition_wait_id": "condition:1",
19201                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19202                        "condition_key": "shared",
19203                        "condition_definition_fingerprint": "sha256:shared",
19204                    }),
19205                ),
19206                history_event(
19207                    "ConditionWaitOpened",
19208                    json!({
19209                        "sequence": 2,
19210                        "condition_wait_id": "condition:2",
19211                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19212                        "condition_key": "shared",
19213                        "condition_definition_fingerprint": "sha256:shared",
19214                    }),
19215                ),
19216                history_event(
19217                    "ConditionWaitSatisfied",
19218                    json!({
19219                        "sequence": 2,
19220                        "condition_wait_id": "condition:2",
19221                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19222                        "condition_key": "shared",
19223                        "condition_definition_fingerprint": "sha256:shared",
19224                    }),
19225                ),
19226            ],
19227            DEFAULT_CODEC,
19228        );
19229
19230        for _cold_worker_or_restart in 0..2 {
19231            let commands = worker()
19232                .execute_workflow_task(task.clone())
19233                .expect("adjacent loop waits replay deterministically");
19234            assert_eq!(commands.len(), 1);
19235            assert_eq!(commands[0]["type"], "complete_workflow");
19236            assert_eq!(
19237                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19238                json!(["satisfied", "satisfied"])
19239            );
19240        }
19241    }
19242
19243    #[test]
19244    fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19245        let history = vec![history_event(
19246            "ConditionWaitOpened",
19247            json!({
19248                "sequence": 12,
19249                "condition_wait_id": "condition:12",
19250                "condition_wait_occurrence_id": "rust:condition-wait:0",
19251                "condition_key": "approval",
19252                "condition_definition_fingerprint": "sha256:approval-v1",
19253                "timeout_seconds": 30,
19254            }),
19255        )];
19256        for (options, expected_reason) in [
19257            (
19258                ConditionWaitOptions::new("changed", "sha256:approval-v1")
19259                    .timeout(Duration::from_secs(30)),
19260                "condition_wait_key_mismatch",
19261            ),
19262            (
19263                ConditionWaitOptions::new("approval", "sha256:approval-v2")
19264                    .timeout(Duration::from_secs(30)),
19265                "condition_wait_predicate_mismatch",
19266            ),
19267            (
19268                ConditionWaitOptions::new("approval", "sha256:approval-v1")
19269                    .timeout(Duration::from_secs(29)),
19270                "condition_wait_timeout_mismatch",
19271            ),
19272        ] {
19273            let ctx = workflow_context(history.clone());
19274            let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19275            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19276            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19277                wait.as_mut().poll(&mut task_context)
19278            else {
19279                panic!("changed condition definition must fail replay");
19280            };
19281            assert_eq!(failure.reason, expected_reason);
19282            assert_eq!(failure.sequence, Some(12));
19283        }
19284    }
19285
19286    #[test]
19287    fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19288        let error = WorkflowState::new(
19289            vec![history_event(
19290                "ConditionWaitOpened",
19291                json!({
19292                    "sequence": 12,
19293                    "condition_wait_id": "condition:12",
19294                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19295                    "condition_key": "approval",
19296                }),
19297            )],
19298            "rust-workers".to_string(),
19299            DEFAULT_CODEC.to_string(),
19300            None,
19301        )
19302        .expect_err("condition history without a predicate fingerprint must fail");
19303
19304        assert!(matches!(
19305            error,
19306            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19307                if reason == "condition_wait_predicate_fingerprint_missing"
19308        ));
19309    }
19310
19311    #[test]
19312    fn condition_wait_history_requires_authored_occurrence_identity() {
19313        let error = WorkflowState::new(
19314            vec![history_event(
19315                "ConditionWaitOpened",
19316                json!({
19317                    "sequence": 12,
19318                    "condition_wait_id": "condition:12",
19319                    "condition_key": "approval",
19320                    "condition_definition_fingerprint": "sha256:approval-v1",
19321                }),
19322            )],
19323            "rust-workers".to_string(),
19324            DEFAULT_CODEC.to_string(),
19325            None,
19326        )
19327        .expect_err("condition history without occurrence identity must fail");
19328
19329        assert!(matches!(
19330            error,
19331            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19332                if reason == "condition_wait_occurrence_id_missing"
19333        ));
19334    }
19335
19336    #[test]
19337    fn typed_search_attribute_updates_validate_emit_and_replay() {
19338        let update = SearchAttributeUpdate::new()
19339            .keyword("OrderStatus", " waiting ")
19340            .expect("keyword")
19341            .int("Attempt", 3)
19342            .expect("int")
19343            .bool("Escalated", false)
19344            .expect("bool")
19345            .keyword_list("Regions", ["us-east", "eu-west"])
19346            .expect("list")
19347            .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19348            .expect("datetime")
19349            .delete("LegacyStatus")
19350            .expect("delete");
19351        let ctx = workflow_context(Vec::new());
19352        ctx.upsert_search_attributes(update.clone())
19353            .expect("typed update");
19354        assert_eq!(
19355            ctx.take_commands().expect("search-attribute command"),
19356            vec![json!({
19357                "type": "upsert_search_attributes",
19358                "attributes": {
19359                    "Attempt": 3,
19360                    "Escalated": false,
19361                    "LegacyStatus": null,
19362                    "OrderStatus": "waiting",
19363                    "Regions": ["us-east", "eu-west"],
19364                    "UpdatedAt": "2026-08-22T04:00:00Z",
19365                },
19366                "attribute_types": {
19367                    "Attempt": "int",
19368                    "Escalated": "bool",
19369                    "OrderStatus": "keyword",
19370                    "Regions": "keyword_list",
19371                    "UpdatedAt": "datetime",
19372                },
19373            })]
19374        );
19375
19376        let replay = workflow_context(vec![history_event(
19377            "SearchAttributesUpserted",
19378            json!({
19379                "sequence": 6,
19380                "attributes": {
19381                    "Attempt": 3,
19382                    "Escalated": false,
19383                    "LegacyStatus": null,
19384                    "OrderStatus": "waiting",
19385                    "Regions": ["us-east", "eu-west"],
19386                    "UpdatedAt": "2026-08-22T04:00:00Z",
19387                },
19388                "attribute_types": {
19389                    "Attempt": "int",
19390                    "Escalated": "bool",
19391                    "OrderStatus": "keyword",
19392                    "Regions": "keyword_list",
19393                    "UpdatedAt": "datetime",
19394                },
19395                "merged": {},
19396            }),
19397        )]);
19398        replay
19399            .upsert_search_attributes(update)
19400            .expect("matching update replays");
19401        assert!(replay.take_commands().expect("commands").is_empty());
19402        replay.ensure_history_consumed().expect("history consumed");
19403
19404        let type_drift = workflow_context(vec![history_event(
19405            "SearchAttributesUpserted",
19406            json!({
19407                "sequence": 7,
19408                "attributes": {"OrderStatus": "waiting"},
19409                "attribute_types": {"OrderStatus": "keyword"},
19410                "merged": {"OrderStatus": "waiting"},
19411            }),
19412        )]);
19413        let error = type_drift
19414            .upsert_search_attributes(
19415                SearchAttributeUpdate::new()
19416                    .string("OrderStatus", "waiting")
19417                    .expect("string update"),
19418            )
19419            .expect_err("same JSON value with a changed type must fail replay");
19420        assert!(matches!(
19421            error,
19422            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19423                if reason == "search_attribute_type_mismatch"
19424        ));
19425
19426        let malformed_types = WorkflowState::new(
19427            vec![history_event(
19428                "SearchAttributesUpserted",
19429                json!({
19430                    "sequence": 8,
19431                    "attributes": {"OrderStatus": "waiting"},
19432                    "attribute_types": {"OrderStatus": "unsupported"},
19433                    "merged": {"OrderStatus": "waiting"},
19434                }),
19435            )],
19436            "rust-workers".to_string(),
19437            DEFAULT_CODEC.to_string(),
19438            None,
19439        )
19440        .expect_err("unsupported search-attribute type metadata must fail");
19441        assert!(matches!(
19442            malformed_types,
19443            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19444                if reason == "search_attribute_types_malformed"
19445        ));
19446
19447        assert!(matches!(
19448            SearchAttributeUpdate::new().keyword("bad key", "value"),
19449            Err(SearchAttributeUpdateError::InvalidKey(_))
19450        ));
19451        assert!(matches!(
19452            SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19453            Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19454        ));
19455        assert!(matches!(
19456            SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19457            Err(SearchAttributeUpdateError::ValueTooLong { .. })
19458        ));
19459        assert!(matches!(
19460            SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19461            Err(SearchAttributeUpdateError::InvalidDateTime(_))
19462        ));
19463        assert!(matches!(
19464            workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19465            Err(Error::InvalidSearchAttributeUpdate(
19466                SearchAttributeUpdateError::Empty
19467            ))
19468        ));
19469    }
19470
19471    #[test]
19472    fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19473        let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19474        let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19475
19476        assert!(SearchAttributeUpdate::new()
19477            .string("AsciiDescription", ascii)
19478            .is_ok());
19479        assert!(SearchAttributeUpdate::new()
19480            .string("Utf8Description", utf8)
19481            .is_ok());
19482        assert!(matches!(
19483            SearchAttributeUpdate::new().string(
19484                "TooLongDescription",
19485                "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19486            ),
19487            Err(SearchAttributeUpdateError::ValueTooLong {
19488                kind: "string",
19489                limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19490                ..
19491            })
19492        ));
19493    }
19494
19495    #[test]
19496    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19497        let lone_fire = WorkflowState::new(
19498            vec![history_event(
19499                "TimerFired",
19500                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19501            )],
19502            "rust-workers".to_string(),
19503            DEFAULT_CODEC.to_string(),
19504            None,
19505        )
19506        .expect_err("TimerFired requires TimerScheduled");
19507        assert!(matches!(
19508            lone_fire,
19509            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19510                if reason == "timer_schedule_missing_or_duplicate"
19511        ));
19512
19513        let wrong_identity = WorkflowState::new(
19514            vec![
19515                history_event(
19516                    "TimerScheduled",
19517                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19518                ),
19519                history_event(
19520                    "TimerFired",
19521                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19522                ),
19523            ],
19524            "rust-workers".to_string(),
19525            DEFAULT_CODEC.to_string(),
19526            None,
19527        )
19528        .expect_err("fire must match scheduled timer identity");
19529        assert!(matches!(
19530            wrong_identity,
19531            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19532                if reason == "timer_identity_mismatch"
19533        ));
19534
19535        let duplicate_fire = WorkflowState::new(
19536            vec![
19537                history_event(
19538                    "TimerScheduled",
19539                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19540                ),
19541                history_event(
19542                    "TimerFired",
19543                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19544                ),
19545                history_event(
19546                    "TimerFired",
19547                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19548                ),
19549            ],
19550            "rust-workers".to_string(),
19551            DEFAULT_CODEC.to_string(),
19552            None,
19553        )
19554        .expect_err("a durable timer cannot fire twice");
19555        assert!(matches!(
19556            duplicate_fire,
19557            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19558                if reason == "duplicate_timer_fire"
19559        ));
19560
19561        let wrong_fired_delay = WorkflowState::new(
19562            vec![
19563                history_event(
19564                    "TimerScheduled",
19565                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19566                ),
19567                history_event(
19568                    "TimerFired",
19569                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19570                ),
19571            ],
19572            "rust-workers".to_string(),
19573            DEFAULT_CODEC.to_string(),
19574            None,
19575        )
19576        .expect_err("timer schedule and fire delays must agree");
19577        assert!(matches!(
19578            wrong_fired_delay,
19579            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19580                if reason == "timer_history_delay_mismatch"
19581        ));
19582    }
19583
19584    #[test]
19585    fn replay_rejects_activity_moved_before_recorded_timer() {
19586        let ctx = workflow_context(vec![
19587            history_event(
19588                "TimerScheduled",
19589                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19590            ),
19591            history_event(
19592                "TimerFired",
19593                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19594            ),
19595            history_event(
19596                "ActivityCompleted",
19597                json!({
19598                    "sequence": 2,
19599                    "activity_type": "after-timer",
19600                    "payload_codec": DEFAULT_CODEC,
19601                    "result": fixture_envelope(json!("done")),
19602                }),
19603            ),
19604        ]);
19605        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19606        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19607
19608        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19609            activity.as_mut().poll(&mut task_context)
19610        else {
19611            panic!("reordered durable command must be rejected");
19612        };
19613        assert_eq!(failure.reason, "recorded_command_mismatch");
19614        assert_eq!(failure.sequence, Some(1));
19615        assert_eq!(failure.expected.as_deref(), Some("timer"));
19616        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19617    }
19618
19619    #[test]
19620    fn workflow_context_emits_a_typed_named_signal_wait() {
19621        let ctx = workflow_context(Vec::new());
19622        let mut signal = Box::pin(ctx.wait_signal("finish"));
19623        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19624
19625        assert!(matches!(
19626            signal.as_mut().poll(&mut task_context),
19627            Poll::Pending
19628        ));
19629        assert_eq!(
19630            ctx.take_commands().expect("signal-wait command"),
19631            vec![json!({
19632                "type": "open_signal_wait",
19633                "signal_name": "finish",
19634            })]
19635        );
19636    }
19637
19638    #[test]
19639    fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19640        let ctx = workflow_context(Vec::new());
19641        let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19642        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19643
19644        let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19645        else {
19646            panic!("runtime-reserved signal should be rejected");
19647        };
19648        assert!(message.contains("reserved by the workflow runtime"));
19649        assert!(ctx.take_commands().expect("commands").is_empty());
19650    }
19651
19652    #[tokio::test]
19653    async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19654        let client = Client::builder("http://127.0.0.1:9")
19655            .build()
19656            .expect("client");
19657        let error = client
19658            .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19659            .await
19660            .expect_err("runtime-reserved signal should be rejected before transport");
19661
19662        assert!(
19663            matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19664        );
19665    }
19666
19667    #[test]
19668    fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19669        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19670            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19671                value.to_string(),
19672            )]))
19673            .expect("message payload");
19674            json!({
19675                "schema": MESSAGE_STREAM_SCHEMA,
19676                "stream_name": "orders",
19677                "message_id": message_id,
19678                "position": position,
19679                "payload_envelope": payload,
19680            })
19681        }
19682
19683        fn opened(sequence: u64) -> HistoryEvent {
19684            history_event(
19685                "SignalWaitOpened",
19686                json!({
19687                    "sequence": sequence,
19688                    "signal_name": MESSAGE_STREAM_SIGNAL,
19689                }),
19690            )
19691        }
19692
19693        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19694            history_event(
19695                "SignalApplied",
19696                json!({
19697                    "sequence": sequence,
19698                    "signal_name": MESSAGE_STREAM_SIGNAL,
19699                    "value": fixture_envelope(json!([delivery])),
19700                }),
19701            )
19702        }
19703
19704        fn received(delivery: Value) -> HistoryEvent {
19705            history_event(
19706                "SignalReceived",
19707                json!({
19708                    "signal_name": MESSAGE_STREAM_SIGNAL,
19709                    "arguments": fixture_envelope(json!([delivery])),
19710                    "payload_codec": DEFAULT_CODEC,
19711                }),
19712            )
19713        }
19714
19715        let client = Client::new("http://127.0.0.1:8080").expect("client");
19716        let mut worker = Worker::new(client, "rust-workers");
19717        worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19718            let messages = ctx.message_stream("orders")?.receive(2).await?;
19719            Ok(json!(messages
19720                .into_iter()
19721                .map(|message| message.message_id)
19722                .collect::<Vec<_>>()))
19723        });
19724
19725        let first = delivery("message-1", 1, "one");
19726        let second = delivery("message-2", 2, "two");
19727        let batch = worker
19728            .execute_workflow_task_decision(workflow_task(
19729                "rust.message-stream-batch",
19730                vec![
19731                    opened(1),
19732                    received(first.clone()),
19733                    applied(1, first.clone()),
19734                    received(first.clone()),
19735                    received(second),
19736                ],
19737                DEFAULT_CODEC,
19738            ))
19739            .expect("worker task consumes the available batch");
19740
19741        assert_eq!(batch.commands.len(), 1);
19742        assert_eq!(batch.commands[0]["type"], "complete_workflow");
19743        assert_eq!(
19744            decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19745                .expect("workflow result"),
19746            json!(["message-1", "message-2"])
19747        );
19748        assert_eq!(
19749            batch.message_stream_cursors,
19750            vec![json!({"stream_name": "orders", "through_position": 2})]
19751        );
19752        assert!(batch.message_stream_waits.is_empty());
19753
19754        let partial = worker
19755            .execute_workflow_task_decision(workflow_task(
19756                "rust.message-stream-batch",
19757                vec![opened(1), received(first.clone()), applied(1, first)],
19758                DEFAULT_CODEC,
19759            ))
19760            .expect("worker task returns without waiting for a missing second item");
19761        assert_eq!(partial.commands.len(), 1);
19762        assert_eq!(partial.commands[0]["type"], "complete_workflow");
19763        assert_eq!(
19764            decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19765                .expect("workflow result"),
19766            json!(["message-1"])
19767        );
19768        assert_eq!(
19769            partial.message_stream_cursors,
19770            vec![json!({"stream_name": "orders", "through_position": 1})]
19771        );
19772        assert!(partial.message_stream_waits.is_empty());
19773    }
19774
19775    #[test]
19776    fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19777        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19778            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19779                value.to_string(),
19780            )]))
19781            .expect("message payload");
19782            json!({
19783                "schema": MESSAGE_STREAM_SCHEMA,
19784                "stream_name": "orders",
19785                "message_id": message_id,
19786                "position": position,
19787                "payload_envelope": payload,
19788            })
19789        }
19790
19791        fn opened(sequence: u64) -> HistoryEvent {
19792            history_event(
19793                "SignalWaitOpened",
19794                json!({
19795                    "sequence": sequence,
19796                    "signal_name": MESSAGE_STREAM_SIGNAL,
19797                }),
19798            )
19799        }
19800
19801        fn received(delivery: Value) -> HistoryEvent {
19802            history_event(
19803                "SignalReceived",
19804                json!({
19805                    "signal_name": MESSAGE_STREAM_SIGNAL,
19806                    "arguments": fixture_envelope(json!([delivery])),
19807                    "payload_codec": DEFAULT_CODEC,
19808                }),
19809            )
19810        }
19811
19812        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19813            history_event(
19814                "SignalApplied",
19815                json!({
19816                    "sequence": sequence,
19817                    "signal_name": MESSAGE_STREAM_SIGNAL,
19818                    "value": fixture_envelope(json!([delivery])),
19819                }),
19820            )
19821        }
19822
19823        let client = Client::new("http://127.0.0.1:8080").expect("client");
19824        let mut worker = Worker::new(client, "rust-workers");
19825        worker.register_workflow(
19826            "rust.message-stream-partial-batches",
19827            |ctx, _input| async move {
19828                let stream = ctx.message_stream("orders")?;
19829                let first = stream.receive(10).await?;
19830                let second = stream.receive(10).await?;
19831                Ok(json!([
19832                    first
19833                        .into_iter()
19834                        .map(|message| message.message_id)
19835                        .collect::<Vec<_>>(),
19836                    second
19837                        .into_iter()
19838                        .map(|message| message.message_id)
19839                        .collect::<Vec<_>>(),
19840                ]))
19841            },
19842        );
19843
19844        let first = delivery("message-1", 1, "one");
19845        let second = delivery("message-2", 2, "two");
19846        let decision = worker
19847            .execute_workflow_task_decision(workflow_task(
19848                "rust.message-stream-partial-batches",
19849                vec![
19850                    opened(1),
19851                    received(first.clone()),
19852                    applied(1, first),
19853                    opened(2),
19854                    received(second.clone()),
19855                    applied(2, second),
19856                ],
19857                DEFAULT_CODEC,
19858            ))
19859            .expect("cold replay preserves both authored receive boundaries");
19860
19861        assert_eq!(decision.commands.len(), 1);
19862        assert_eq!(decision.commands[0]["type"], "complete_workflow");
19863        assert_eq!(
19864            decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19865                .expect("workflow result"),
19866            json!([["message-1"], ["message-2"]])
19867        );
19868        assert_eq!(
19869            decision.message_stream_cursors,
19870            vec![json!({"stream_name": "orders", "through_position": 2})]
19871        );
19872        assert!(decision.message_stream_waits.is_empty());
19873    }
19874
19875    #[test]
19876    fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19877        let ctx = workflow_context(Vec::new());
19878        let stream = ctx.message_stream("orders").expect("message stream");
19879        let mut receive = Box::pin(stream.receive(10));
19880        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19881
19882        assert!(matches!(
19883            receive.as_mut().poll(&mut task_context),
19884            Poll::Pending
19885        ));
19886        assert_eq!(
19887            ctx.take_commands().expect("message-stream wait command"),
19888            vec![json!({
19889                "type": "open_signal_wait",
19890                "signal_name": MESSAGE_STREAM_SIGNAL,
19891            })]
19892        );
19893        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19894        assert!(cursors.is_empty());
19895        assert_eq!(
19896            waits,
19897            vec![json!({"stream_name": "orders", "after_position": 0})]
19898        );
19899    }
19900
19901    #[test]
19902    fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
19903        let ctx = workflow_context(vec![history_event(
19904            "SignalReceived",
19905            json!({
19906                "signal_name": MESSAGE_STREAM_SIGNAL,
19907                "arguments": fixture_envelope(json!([{
19908                    "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
19909                    "stream_name": "orders",
19910                    "through_position": 2,
19911                }])),
19912                "payload_codec": DEFAULT_CODEC,
19913            }),
19914        )]);
19915        let stream = ctx.message_stream("orders").expect("message stream");
19916        let mut receive = Box::pin(stream.receive(10));
19917        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19918
19919        assert!(matches!(
19920            receive.as_mut().poll(&mut task_context),
19921            Poll::Pending
19922        ));
19923        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19924        assert_eq!(
19925            cursors,
19926            vec![json!({"stream_name": "orders", "through_position": 2})]
19927        );
19928        assert_eq!(
19929            waits,
19930            vec![json!({"stream_name": "orders", "after_position": 2})]
19931        );
19932    }
19933
19934    #[test]
19935    fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
19936        let mut empty_map = BTreeMap::new();
19937        let mut nested = BTreeMap::new();
19938        nested.insert(
19939            "value".to_string(),
19940            AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
19941        );
19942        let values = vec![
19943            AvroValue::Bytes(vec![0, 255]),
19944            AvroValue::Long(1),
19945            AvroValue::Double(1.0),
19946            AvroValue::Array(Vec::new()),
19947            AvroValue::Map(std::mem::take(&mut empty_map)),
19948            AvroValue::Map(nested),
19949        ];
19950        let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
19951        let transport = vec![json!({
19952            "schema": MESSAGE_STREAM_SCHEMA,
19953            "stream_name": "orders",
19954            "message_id": "message-1",
19955            "position": 1,
19956            "payload_envelope": payload,
19957        })];
19958
19959        for _ in 0..2 {
19960            let Some(MessageStreamDelivery::Message(message)) =
19961                decode_message_stream_delivery(transport.clone()).expect("delivery")
19962            else {
19963                panic!("message delivery expected");
19964            };
19965            assert_eq!(message.arguments, values);
19966            assert!(matches!(message.arguments[1], AvroValue::Long(1)));
19967            assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
19968        }
19969    }
19970
19971    #[test]
19972    fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
19973        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19974            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19975                value.to_string(),
19976            )]))
19977            .expect("message payload");
19978            json!({
19979                "schema": MESSAGE_STREAM_SCHEMA,
19980                "stream_name": "orders",
19981                "message_id": message_id,
19982                "position": position,
19983                "payload_envelope": payload,
19984            })
19985        }
19986
19987        fn opened(sequence: u64) -> HistoryEvent {
19988            history_event(
19989                "SignalWaitOpened",
19990                json!({
19991                    "sequence": sequence,
19992                    "signal_name": MESSAGE_STREAM_SIGNAL,
19993                }),
19994            )
19995        }
19996
19997        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19998            history_event(
19999                "SignalApplied",
20000                json!({
20001                    "sequence": sequence,
20002                    "signal_name": MESSAGE_STREAM_SIGNAL,
20003                    "value": fixture_envelope(json!([delivery])),
20004                }),
20005            )
20006        }
20007
20008        fn worker() -> Worker {
20009            let client = Client::new("http://127.0.0.1:8080").expect("client");
20010            let mut worker = Worker::new(client, "rust-workers");
20011            worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20012                let stream = ctx.message_stream("orders")?;
20013                let first = stream.receive_one().await?;
20014                let second = stream.receive_one().await?;
20015                Ok(json!([first.message_id, second.message_id]))
20016            });
20017            worker
20018        }
20019
20020        fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20021            let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20022            task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20023            task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20024            task
20025        }
20026
20027        let waiting = worker()
20028            .execute_workflow_task_decision(workflow_task(
20029                "rust.message-stream",
20030                Vec::new(),
20031                DEFAULT_CODEC,
20032            ))
20033            .expect("first worker opens the stream wait");
20034        assert_eq!(
20035            waiting.commands,
20036            vec![json!({
20037                "type": "open_signal_wait",
20038                "signal_name": MESSAGE_STREAM_SIGNAL,
20039            })]
20040        );
20041        assert!(waiting.message_stream_cursors.is_empty());
20042        assert_eq!(
20043            waiting.message_stream_waits,
20044            vec![json!({"stream_name": "orders", "after_position": 0})]
20045        );
20046
20047        let first_delivery = delivery("message-1", 1, "one");
20048        let first_arrival = worker()
20049            .execute_workflow_task_decision(task_with_resume(
20050                vec![opened(1)],
20051                first_delivery.clone(),
20052            ))
20053            .expect("replacement worker consumes the first arrival");
20054        assert_eq!(
20055            first_arrival.commands,
20056            vec![json!({
20057                "type": "open_signal_wait",
20058                "signal_name": MESSAGE_STREAM_SIGNAL,
20059            })]
20060        );
20061        assert_eq!(
20062            first_arrival.message_stream_cursors,
20063            vec![json!({"stream_name": "orders", "through_position": 1})]
20064        );
20065        assert_eq!(
20066            first_arrival.message_stream_waits,
20067            vec![json!({"stream_name": "orders", "after_position": 1})]
20068        );
20069
20070        let second_delivery = delivery("message-2", 2, "two");
20071        let first_applied = applied(1, first_delivery);
20072        let completed = worker()
20073            .execute_workflow_task_decision(task_with_resume(
20074                vec![opened(1), first_applied.clone(), opened(2)],
20075                second_delivery.clone(),
20076            ))
20077            .expect("next replacement worker consumes the second arrival");
20078        assert_eq!(completed.commands.len(), 1);
20079        assert_eq!(completed.commands[0]["type"], "complete_workflow");
20080        assert_eq!(
20081            decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20082                .expect("workflow result"),
20083            json!(["message-1", "message-2"])
20084        );
20085        assert_eq!(
20086            completed.message_stream_cursors,
20087            vec![json!({"stream_name": "orders", "through_position": 2})]
20088        );
20089        assert!(completed.message_stream_waits.is_empty());
20090
20091        let replay_history = vec![
20092            opened(1),
20093            first_applied,
20094            opened(2),
20095            applied(2, second_delivery),
20096        ];
20097        for _cold_worker_or_restart in 0..2 {
20098            let replayed = worker()
20099                .execute_workflow_task_decision(workflow_task(
20100                    "rust.message-stream",
20101                    replay_history.clone(),
20102                    DEFAULT_CODEC,
20103                ))
20104                .expect("cold worker replays each logical message exactly once");
20105            assert_eq!(replayed.commands.len(), 1);
20106            assert_eq!(
20107                decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20108                    .expect("replayed workflow result"),
20109                json!(["message-1", "message-2"])
20110            );
20111            assert_eq!(
20112                replayed.message_stream_cursors,
20113                vec![json!({"stream_name": "orders", "through_position": 2})]
20114            );
20115            assert!(replayed.message_stream_waits.is_empty());
20116        }
20117    }
20118
20119    #[test]
20120    fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20121        assert!(!worker_protocol_supports_message_streams("1.14"));
20122        assert!(worker_protocol_supports_message_streams("1.15"));
20123        assert!(worker_protocol_supports_message_streams("1.16"));
20124        assert!(worker_protocol_supports_message_streams(
20125            WORKER_PROTOCOL_VERSION
20126        ));
20127        assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20128    }
20129
20130    #[test]
20131    fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20132        let ctx = workflow_context(vec![
20133            history_event(
20134                "ConditionWaitOpened",
20135                json!({
20136                    "sequence": 1,
20137                    "condition_wait_id": "condition:1",
20138                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20139                    "condition_key": "signal:finish",
20140                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20141                }),
20142            ),
20143            history_event(
20144                "ConditionWaitSatisfied",
20145                json!({
20146                    "sequence": 1,
20147                    "condition_wait_id": "condition:1",
20148                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20149                    "condition_key": "signal:finish",
20150                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20151                }),
20152            ),
20153            history_event(
20154                "SignalReceived",
20155                json!({"signal_name": "finish", "arguments": []}),
20156            ),
20157        ]);
20158        let mut signal = Box::pin(ctx.wait_signal("finish"));
20159        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20160
20161        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20162            signal.as_mut().poll(&mut task_context)
20163        else {
20164            panic!("condition history must not resolve as a typed signal wait");
20165        };
20166        assert_eq!(failure.reason, "recorded_command_mismatch");
20167        assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20168    }
20169
20170    #[test]
20171    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20172        let signal_then_timer = vec![
20173            history_event(
20174                "SignalWaitOpened",
20175                json!({"sequence": 1, "signal_name": "go"}),
20176            ),
20177            history_event(
20178                "SignalApplied",
20179                json!({
20180                    "sequence": 1,
20181                    "signal_name": "go",
20182                    "value": fixture_envelope(json!(["now"])),
20183                }),
20184            ),
20185            history_event(
20186                "TimerScheduled",
20187                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20188            ),
20189            history_event(
20190                "TimerFired",
20191                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20192            ),
20193        ];
20194
20195        let ctx = workflow_context(signal_then_timer.clone());
20196        let mut signal = Box::pin(ctx.wait_signal("go"));
20197        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20198        assert!(matches!(
20199            signal.as_mut().poll(&mut task_context),
20200            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20201        ));
20202        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20203        assert!(matches!(
20204            timer.as_mut().poll(&mut task_context),
20205            Poll::Ready(Ok(()))
20206        ));
20207        ctx.ensure_history_consumed()
20208            .expect("signal and timer history consumed in order");
20209
20210        let reordered = workflow_context(signal_then_timer);
20211        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20212        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20213            timer_first.as_mut().poll(&mut task_context)
20214        else {
20215            panic!("timer cannot consume signal-wait-first history");
20216        };
20217        assert_eq!(failure.reason, "recorded_command_mismatch");
20218        assert_eq!(failure.sequence, Some(1));
20219        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20220
20221        let timer_then_signal = vec![
20222            history_event(
20223                "TimerScheduled",
20224                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20225            ),
20226            history_event(
20227                "TimerFired",
20228                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20229            ),
20230            history_event(
20231                "SignalWaitOpened",
20232                json!({"sequence": 2, "signal_name": "go"}),
20233            ),
20234            history_event(
20235                "SignalApplied",
20236                json!({
20237                    "sequence": 2,
20238                    "signal_name": "go",
20239                    "value": fixture_envelope(json!([])),
20240                }),
20241            ),
20242        ];
20243        let reordered = workflow_context(timer_then_signal);
20244        let mut signal_first = Box::pin(reordered.wait_signal("go"));
20245        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20246            signal_first.as_mut().poll(&mut task_context)
20247        else {
20248            panic!("signal wait cannot consume timer-first history");
20249        };
20250        assert_eq!(failure.reason, "recorded_command_mismatch");
20251        assert_eq!(failure.sequence, Some(1));
20252        assert_eq!(failure.expected.as_deref(), Some("timer"));
20253    }
20254
20255    #[test]
20256    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20257        let duplicate_timer = WorkflowState::new(
20258            vec![
20259                history_event(
20260                    "TimerScheduled",
20261                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20262                ),
20263                history_event(
20264                    "TimerScheduled",
20265                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20266                ),
20267            ],
20268            "rust-workers".to_string(),
20269            DEFAULT_CODEC.to_string(),
20270            None,
20271        )
20272        .expect_err("one workflow sequence cannot schedule two timers");
20273        assert!(matches!(
20274            duplicate_timer,
20275            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20276                if reason == "timer_schedule_missing_or_duplicate"
20277        ));
20278
20279        let colliding_kinds = WorkflowState::new(
20280            vec![
20281                history_event(
20282                    "TimerScheduled",
20283                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20284                ),
20285                history_event(
20286                    "ActivityCompleted",
20287                    json!({"sequence": 1, "activity_type": "same-sequence"}),
20288                ),
20289            ],
20290            "rust-workers".to_string(),
20291            DEFAULT_CODEC.to_string(),
20292            None,
20293        )
20294        .expect_err("one workflow sequence cannot identify two command kinds");
20295        assert!(matches!(
20296            colliding_kinds,
20297            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20298                if reason == "durable_command_sequence_collision"
20299        ));
20300
20301        let duplicate_signal_wait = WorkflowState::new(
20302            vec![
20303                history_event(
20304                    "SignalWaitOpened",
20305                    json!({"sequence": 1, "signal_name": "go"}),
20306                ),
20307                history_event(
20308                    "SignalWaitOpened",
20309                    json!({"sequence": 1, "signal_name": "go"}),
20310                ),
20311            ],
20312            "rust-workers".to_string(),
20313            DEFAULT_CODEC.to_string(),
20314            None,
20315        )
20316        .expect_err("one workflow sequence cannot open two signal waits");
20317        assert!(matches!(
20318            duplicate_signal_wait,
20319            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20320                if reason == "signal_wait_open_missing_or_duplicate"
20321        ));
20322    }
20323
20324    #[test]
20325    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20326        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20327            .expect("side-effect result");
20328        let ctx = workflow_context(vec![history_event(
20329            "SideEffectRecorded",
20330            json!({"sequence": 99, "result": result}),
20331        )]);
20332
20333        let replayed: Value = ctx
20334            .side_effect(|| panic!("recorded side effect must not run"))
20335            .expect("positive global workflow sequence is valid");
20336        assert_eq!(replayed, json!({"captured": true}));
20337        ctx.ensure_history_consumed().expect("history consumed");
20338    }
20339
20340    #[test]
20341    fn workflow_history_rejects_zero_and_descending_command_sequences() {
20342        let result =
20343            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20344        let zero = WorkflowState::new(
20345            vec![history_event(
20346                "SideEffectRecorded",
20347                json!({"sequence": 0, "result": result.clone()}),
20348            )],
20349            "rust-workers".to_string(),
20350            DEFAULT_CODEC.to_string(),
20351            None,
20352        )
20353        .expect_err("durable command sequences must be positive");
20354        assert!(matches!(
20355            zero,
20356            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20357                if reason == "durable_command_sequence_invalid"
20358        ));
20359
20360        let descending = WorkflowState::new(
20361            vec![
20362                history_event(
20363                    "SideEffectRecorded",
20364                    json!({"sequence": 3, "result": result}),
20365                ),
20366                history_event(
20367                    "VersionMarkerRecorded",
20368                    json!({
20369                        "sequence": 2,
20370                        "change_id": "descending-marker",
20371                        "version": 1,
20372                        "min_supported": 1,
20373                        "max_supported": 1,
20374                    }),
20375                ),
20376            ],
20377            "rust-workers".to_string(),
20378            DEFAULT_CODEC.to_string(),
20379            None,
20380        )
20381        .expect_err("new durable commands must remain strictly ordered");
20382        let Error::NonDeterministicReplay(failure) = descending else {
20383            panic!("expected typed replay failure");
20384        };
20385        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20386        assert_eq!(failure.sequence, Some(2));
20387        assert_eq!(
20388            failure.expected.as_deref(),
20389            Some("workflow sequence greater than 3")
20390        );
20391        assert_eq!(failure.actual.as_deref(), Some("2"));
20392    }
20393
20394    #[test]
20395    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20396        fn worker() -> Worker {
20397            let client = Client::new("http://127.0.0.1:8080").expect("client");
20398            let mut worker = Worker::new(client, "rust-workers");
20399            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20400                ctx.wait_signal("finish").await?;
20401                let marker: String =
20402                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20403                assert_eq!(marker, "after-finish");
20404                Ok(json!("finished"))
20405            });
20406            worker
20407        }
20408
20409        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20410            .expect("side-effect result");
20411        let task = workflow_task(
20412            "rust.finish-after-gaps",
20413            vec![
20414                history_event(
20415                    "SignalWaitOpened",
20416                    json!({"sequence": 1, "signal_name": "finish"}),
20417                ),
20418                history_event(
20419                    "SignalReceived",
20420                    json!({
20421                        "signal_id": "increment-3",
20422                        "signal_name": "increment",
20423                        "workflow_sequence": 2,
20424                        "payload_codec": DEFAULT_CODEC,
20425                        "arguments": fixture_envelope(json!([3])),
20426                    }),
20427                ),
20428                history_event(
20429                    "SignalReceived",
20430                    json!({
20431                        "signal_id": "increment-5",
20432                        "signal_name": "increment",
20433                        "workflow_sequence": 3,
20434                        "payload_codec": DEFAULT_CODEC,
20435                        "arguments": fixture_envelope(json!([5])),
20436                    }),
20437                ),
20438                history_event(
20439                    "SignalReceived",
20440                    json!({
20441                        "signal_id": "finish",
20442                        "signal_name": "finish",
20443                        "workflow_sequence": 4,
20444                        "payload_codec": DEFAULT_CODEC,
20445                        "arguments": fixture_envelope(json!([])),
20446                    }),
20447                ),
20448                history_event(
20449                    "SignalApplied",
20450                    json!({
20451                        "sequence": 1,
20452                        "signal_id": "finish",
20453                        "signal_name": "finish",
20454                        "payload_codec": DEFAULT_CODEC,
20455                        "value": fixture_envelope(json!([])),
20456                    }),
20457                ),
20458                history_event(
20459                    "SideEffectRecorded",
20460                    json!({"sequence": 5, "result": marker}),
20461                ),
20462            ],
20463            DEFAULT_CODEC,
20464        );
20465
20466        for _original_or_cold_worker in 0..2 {
20467            let commands = worker()
20468                .execute_workflow_task(task.clone())
20469                .expect("signal gaps preserve deterministic replay");
20470            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20471            assert_eq!(commands[0]["type"], "complete_workflow");
20472            assert_eq!(
20473                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20474                json!("finished")
20475            );
20476        }
20477    }
20478
20479    #[test]
20480    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20481        let ctx = workflow_context(Vec::new());
20482        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20483        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20484        assert!(matches!(
20485            sleep.as_mut().poll(&mut task_context),
20486            Poll::Ready(Err(Error::TimerDurationOverflow))
20487        ));
20488        assert!(ctx.take_commands().expect("commands").is_empty());
20489    }
20490
20491    #[test]
20492    fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20493        let entries = AvroValue::Map(BTreeMap::from([
20494            ("text".to_string(), AvroValue::String("same".to_string())),
20495            (
20496                "nested".to_string(),
20497                AvroValue::Map(BTreeMap::from([
20498                    ("beta".to_string(), AvroValue::Long(2)),
20499                    ("alpha".to_string(), AvroValue::Long(1)),
20500                ])),
20501            ),
20502            ("long".to_string(), AvroValue::Long(7)),
20503            ("double".to_string(), AvroValue::Double(7.0)),
20504            ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20505        ]));
20506        let ctx = workflow_context(Vec::new());
20507        ctx.upsert_memo(entries.clone()).expect("valid memo update");
20508        let commands = ctx.take_commands().expect("commands");
20509
20510        assert_eq!(commands.len(), 1);
20511        assert_eq!(commands[0]["type"], "upsert_memo");
20512        let server_entries = json!({
20513            "codec": "avro",
20514            "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20515        });
20516        assert_eq!(
20517            commands[0]["entries"]
20518                .as_object()
20519                .expect("entries envelope")
20520                .keys()
20521                .collect::<Vec<_>>(),
20522            vec!["blob", "codec"]
20523        );
20524        assert_eq!(commands[0]["entries"], server_entries);
20525        let wire_entries =
20526            decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20527        assert_eq!(wire_entries, entries);
20528
20529        let history = vec![history_event(
20530            "MemoUpserted",
20531            json!({
20532                "sequence": 1,
20533                "entries": server_entries.clone(),
20534                "merged": server_entries,
20535            }),
20536        )];
20537        let replay = workflow_context(history.clone());
20538        replay
20539            .upsert_memo(entries.clone())
20540            .expect("matching replay identity");
20541        assert!(replay.take_commands().expect("replay commands").is_empty());
20542
20543        let changed_types = AvroValue::Map(BTreeMap::from([
20544            ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20545            (
20546                "nested".to_string(),
20547                AvroValue::Map(BTreeMap::from([
20548                    ("alpha".to_string(), AvroValue::Long(1)),
20549                    ("beta".to_string(), AvroValue::Long(2)),
20550                ])),
20551            ),
20552            ("long".to_string(), AvroValue::Double(7.0)),
20553            ("double".to_string(), AvroValue::Long(7)),
20554            ("binary".to_string(), AvroValue::String("same".to_string())),
20555        ]));
20556        let error = workflow_context(history)
20557            .upsert_memo(changed_types)
20558            .expect_err("memo replay identity must preserve Avro value types");
20559        assert!(matches!(
20560            error,
20561            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20562        ));
20563    }
20564
20565    #[test]
20566    fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20567        let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20568            .expect("memo envelope");
20569        let replay = workflow_context(vec![history_event(
20570            "MemoUpserted",
20571            json!({
20572                "sequence": 1,
20573                "entries": original.clone(),
20574                "merged": original
20575            }),
20576        )]);
20577        let error = replay
20578            .upsert_memo(json!({"stage": "changed"}))
20579            .expect_err("changed memo update must fail replay");
20580        assert!(matches!(
20581            error,
20582            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20583        ));
20584
20585        let invalid = workflow_context(Vec::new())
20586            .upsert_memo(
20587                json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20588            )
20589            .expect_err("oversized key");
20590        assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20591    }
20592
20593    #[test]
20594    fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20595        let negative_zero = AvroValue::Map(BTreeMap::from([(
20596            "reading".to_string(),
20597            AvroValue::Double(-0.0),
20598        )]));
20599        let negative_zero_envelope =
20600            encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20601        let history = vec![history_event(
20602            "MemoUpserted",
20603            json!({
20604                "sequence": 1,
20605                "entries": negative_zero_envelope.clone(),
20606                "merged": negative_zero_envelope,
20607            }),
20608        )];
20609
20610        workflow_context(history.clone())
20611            .upsert_memo(negative_zero)
20612            .expect("matching negative-zero history identity");
20613
20614        let error = workflow_context(history)
20615            .upsert_memo(AvroValue::Map(BTreeMap::from([(
20616                "reading".to_string(),
20617                AvroValue::Double(0.0),
20618            )])))
20619            .expect_err("positive zero must not consume negative-zero memo history");
20620        assert!(matches!(
20621            error,
20622            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20623        ));
20624    }
20625
20626    #[test]
20627    fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20628        let supported = json!({
20629            "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20630            "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20631        });
20632        assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20633        assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20634            "workflow_memo_updates": {"supported": false},
20635            "supported_workflow_task_commands": ["upsert_memo"]
20636        }))));
20637        assert!(commands_use_workflow_memo_updates(&[json!({
20638            "type": "upsert_memo",
20639            "entries": {"stage": "processing"}
20640        })]));
20641    }
20642
20643    #[test]
20644    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20645        let client = Client::new("http://127.0.0.1:8080").expect("client");
20646        let mut worker = Worker::new(client, "rust-workers");
20647        worker.register_workflow("rust.timer", |ctx, _input| async move {
20648            ctx.sleep(Duration::from_secs(5)).await?;
20649            ctx.activity("after-timer", json!([])).await
20650        });
20651
20652        let task = |history_events| WorkflowTask {
20653            task_id: "wft-rust-timer-1".to_string(),
20654            workflow_command_id: None,
20655            workflow_id: Some("wf-rust-timer".to_string()),
20656            run_id: Some("run-rust-timer".to_string()),
20657            workflow_type: "rust.timer".to_string(),
20658            cancel_requested: false,
20659            payload_codec: DEFAULT_CODEC.to_string(),
20660            arguments: Some(
20661                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20662            ),
20663            history_events,
20664            total_history_events: None,
20665            history_size_bytes: None,
20666            continue_as_new_recommended: None,
20667            history_budget_pressure: None,
20668            next_history_page_token: None,
20669            workflow_task_attempt: 1,
20670            workflow_signal_id: None,
20671            signal_name: None,
20672            signal_arguments: None,
20673            workflow_update_id: None,
20674            update_name: None,
20675            lease_owner: Some("rust-worker".to_string()),
20676        };
20677
20678        let initial = worker
20679            .execute_workflow_task(task(Vec::new()))
20680            .expect("initial timer task");
20681        assert_eq!(
20682            initial,
20683            vec![json!({"type": "start_timer", "delay_seconds": 5})]
20684        );
20685
20686        let activity_result =
20687            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20688        let replayed = worker
20689            .execute_workflow_task(task(vec![
20690                history_event(
20691                    "TimerScheduled",
20692                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20693                ),
20694                history_event(
20695                    "TimerFired",
20696                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20697                ),
20698                history_event(
20699                    "ActivityCompleted",
20700                    json!({
20701                        "sequence": 2,
20702                        "activity_type": "after-timer",
20703                        "payload_codec": DEFAULT_CODEC,
20704                        "result": activity_result,
20705                    }),
20706                ),
20707            ]))
20708            .expect("replayed workflow task");
20709        assert_eq!(replayed.len(), 1);
20710        assert_eq!(replayed[0]["type"], "complete_workflow");
20711        assert_eq!(
20712            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20713            json!("done")
20714        );
20715    }
20716
20717    #[test]
20718    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20719        let client = Client::new("http://127.0.0.1:8080").expect("client");
20720        let mut worker = Worker::new(client, "rust-workers");
20721        worker.register_workflow("rust.continue", |ctx, _input| async move {
20722            ctx.continue_as_new_with_options(
20723                ContinueAsNewOptions::new()
20724                    .workflow_type("rust.next")
20725                    .task_queue("next-workers"),
20726                json!([2, {"cursor": "next"}]),
20727            )
20728        });
20729
20730        let commands = worker
20731            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20732            .expect("continue-as-new command");
20733
20734        assert_eq!(commands.len(), 1);
20735        assert_eq!(commands[0]["type"], "continue_as_new");
20736        assert_eq!(commands[0]["workflow_type"], "rust.next");
20737        assert_eq!(commands[0]["queue"], "next-workers");
20738        assert_eq!(
20739            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20740                .expect("continue-as-new arguments"),
20741            json!([2, {"cursor": "next"}])
20742        );
20743    }
20744
20745    #[test]
20746    fn continue_as_new_preserves_typed_arguments() {
20747        let client = Client::new("http://127.0.0.1:8080").expect("client");
20748        let mut worker = Worker::new(client, "rust-workers");
20749        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20750            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20751            unreachable!("continue-as-new returns a control-flow error")
20752        });
20753
20754        let commands = worker
20755            .execute_workflow_task(workflow_task(
20756                "rust.typed-continue",
20757                Vec::new(),
20758                DEFAULT_CODEC,
20759            ))
20760            .expect("typed continue-as-new command");
20761
20762        assert_eq!(commands[0]["type"], "continue_as_new");
20763        assert_eq!(
20764            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20765                .expect("typed continue arguments"),
20766            AvroValue::Array(vec![typed_fidelity_probe()])
20767        );
20768    }
20769
20770    #[test]
20771    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20772        let client = Client::new("http://127.0.0.1:8080").expect("client");
20773        let mut worker = Worker::new(client, "rust-workers");
20774        worker.register_workflow("rust.continue", |ctx, _input| async move {
20775            ctx.continue_as_new(json!([2]))
20776        });
20777        let task = workflow_task(
20778            "rust.continue",
20779            vec![history_event(
20780                "WorkflowContinuedAsNew",
20781                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20782            )],
20783            DEFAULT_CODEC,
20784        );
20785
20786        for _worker_restart_or_redelivery in 0..2 {
20787            let commands = worker
20788                .execute_workflow_task(task.clone())
20789                .expect("recorded transition replays");
20790            assert!(
20791                commands.is_empty(),
20792                "replay must not emit another successor"
20793            );
20794        }
20795    }
20796
20797    #[test]
20798    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20799        let ctx = workflow_context(Vec::new());
20800        let error = ctx
20801            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
20802            .expect_err("blank queue must be rejected");
20803
20804        let Error::InvalidContinueAsNewOptions(error) = error else {
20805            panic!("expected typed continue-as-new validation error");
20806        };
20807        assert_eq!(error.field, "task_queue");
20808        assert!(ctx.take_commands().expect("commands").is_empty());
20809    }
20810
20811    #[test]
20812    fn workflow_context_exposes_server_history_budget() {
20813        let client = Client::new("http://127.0.0.1:8080").expect("client");
20814        let mut worker = Worker::new(client, "rust-workers");
20815        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20816            let budget = ctx.history_budget()?;
20817            Ok(json!({
20818                "events": budget.event_count,
20819                "bytes": budget.size_bytes,
20820                "recommended": budget.continue_as_new_recommended,
20821                "pressure": budget.pressure,
20822            }))
20823        });
20824        let task: WorkflowTask = serde_json::from_value(json!({
20825            "task_id": "task-history-budget",
20826            "workflow_type": "rust.history-budget",
20827            "payload_codec": DEFAULT_CODEC,
20828            "history_events": [],
20829            "total_history_events": 480,
20830            "history_size_bytes": 1_048_576,
20831            "continue_as_new_recommended": true,
20832            "history_budget_pressure": "continue_as_new_recommended",
20833        }))
20834        .expect("published workflow task");
20835
20836        let commands = worker
20837            .execute_workflow_task(task)
20838            .expect("history-budget workflow");
20839        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20840        assert_eq!(result["events"], 480);
20841        assert_eq!(result["bytes"], 1_048_576);
20842        assert_eq!(result["recommended"], true);
20843        assert_eq!(result["pressure"], "continue_as_new_recommended");
20844    }
20845
20846    #[test]
20847    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20848        let client = Client::new("http://127.0.0.1:8080").expect("client");
20849        let mut worker = Worker::new(client, "rust-workers");
20850        worker.register_workflow("rust.failing", |_ctx, _input| async move {
20851            Err(Error::Codec("rust_conformance_failure".to_string()))
20852        });
20853        let task = WorkflowTask {
20854            task_id: "wft-rust-failing-1".to_string(),
20855            workflow_command_id: None,
20856            workflow_id: Some("wf-rust-failing".to_string()),
20857            run_id: Some("run-rust-failing".to_string()),
20858            workflow_type: "rust.failing".to_string(),
20859            cancel_requested: false,
20860            payload_codec: DEFAULT_CODEC.to_string(),
20861            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20862            history_events: Vec::new(),
20863            total_history_events: Some(0),
20864            history_size_bytes: None,
20865            continue_as_new_recommended: None,
20866            history_budget_pressure: None,
20867            next_history_page_token: None,
20868            workflow_task_attempt: 1,
20869            workflow_signal_id: None,
20870            signal_name: None,
20871            signal_arguments: None,
20872            workflow_update_id: None,
20873            update_name: None,
20874            lease_owner: Some("rust-worker".to_string()),
20875        };
20876
20877        let commands = worker
20878            .execute_workflow_task(task)
20879            .expect("handler failure becomes a workflow command");
20880
20881        assert_eq!(commands.len(), 1);
20882        assert_eq!(commands[0]["type"], "fail_workflow");
20883        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20884        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20885        assert_eq!(commands[0]["non_retryable"], false);
20886        assert_eq!(
20887            commands[0]["message"],
20888            "codec error: rust_conformance_failure"
20889        );
20890        assert_eq!(
20891            commands[0]["exception"]["message"],
20892            "codec error: rust_conformance_failure"
20893        );
20894    }
20895
20896    #[test]
20897    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
20898        let client = Client::new("http://127.0.0.1:8080").expect("client");
20899        let mut worker = Worker::new(client, "rust-workers");
20900        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
20901            let _: String = ctx.side_effect(|| "captured".to_string())?;
20902            Err(Error::WorkerLoop("application failure".to_string()))
20903        });
20904
20905        let commands = worker
20906            .execute_workflow_task(workflow_task(
20907                "rust.failing-after-side-effect",
20908                Vec::new(),
20909                DEFAULT_CODEC,
20910            ))
20911            .expect("ordinary failure remains a workflow decision");
20912
20913        assert_eq!(commands.len(), 2);
20914        assert_eq!(commands[0]["type"], "record_side_effect");
20915        assert_eq!(commands[1]["type"], "fail_workflow");
20916    }
20917
20918    #[test]
20919    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
20920        let client = Client::new("http://127.0.0.1:8080").expect("client");
20921        let mut worker = Worker::new(client, "rust-workers");
20922        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
20923            Err(Error::WorkerLoop("application failure".to_string()))
20924        });
20925        let result =
20926            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
20927
20928        let error = worker
20929            .execute_workflow_task(workflow_task(
20930                "rust.removed-side-effect",
20931                vec![history_event(
20932                    "SideEffectRecorded",
20933                    json!({"sequence": 1, "result": result}),
20934                )],
20935                DEFAULT_CODEC,
20936            ))
20937            .expect_err("removed committed history must not become fail_workflow");
20938
20939        let Error::NonDeterministicReplay(failure) = error else {
20940            panic!("expected typed replay failure");
20941        };
20942        assert_eq!(failure.reason, "recorded_commands_unconsumed");
20943        assert_eq!(failure.sequence, Some(1));
20944        assert_eq!(failure.expected.as_deref(), Some("side effect"));
20945    }
20946
20947    #[test]
20948    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
20949        let client = Client::new("http://127.0.0.1:8080").expect("client");
20950        let mut worker = Worker::new(client, "rust-workers");
20951        worker.register_workflow(
20952            "rust.side-effect-before-marker-error",
20953            |ctx, _input| async move {
20954                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
20955                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
20956                ctx.get_version("restart-safe", 2, 2)?;
20957                Ok(Value::Null)
20958            },
20959        );
20960
20961        let error = worker
20962            .execute_workflow_task(workflow_task(
20963                "rust.side-effect-before-marker-error",
20964                vec![history_event(
20965                    "VersionMarkerRecorded",
20966                    json!({
20967                        "sequence": 1,
20968                        "change_id": "restart-safe",
20969                        "version": 1,
20970                        "min_supported": 1,
20971                        "max_supported": 1,
20972                    }),
20973                )],
20974                DEFAULT_CODEC,
20975            ))
20976            .expect_err("replay error must return no queued workflow commands");
20977
20978        let Error::NonDeterministicReplay(failure) = error else {
20979            panic!("expected typed replay failure");
20980        };
20981        assert_eq!(failure.reason, "version_marker_incompatible_range");
20982        assert_eq!(failure.sequence, Some(1));
20983    }
20984
20985    #[test]
20986    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
20987        let client = Client::new("http://127.0.0.1:8080").expect("client");
20988        let mut worker = Worker::new(client, "rust-workers");
20989        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
20990            ctx.sleep(Duration::from_secs(5)).await?;
20991            Ok(json!({"status": "timer fired"}))
20992        });
20993
20994        let task = WorkflowTask {
20995            task_id: "wft-rust-timer-pending".to_string(),
20996            workflow_command_id: None,
20997            workflow_id: Some("wf-rust-timer".to_string()),
20998            run_id: Some("run-rust-timer".to_string()),
20999            workflow_type: "rust.timer.pending".to_string(),
21000            cancel_requested: false,
21001            payload_codec: DEFAULT_CODEC.to_string(),
21002            arguments: Some(
21003                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21004            ),
21005            history_events: vec![history_event(
21006                "TimerScheduled",
21007                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21008            )],
21009            total_history_events: Some(1),
21010            history_size_bytes: None,
21011            continue_as_new_recommended: None,
21012            history_budget_pressure: None,
21013            next_history_page_token: None,
21014            workflow_task_attempt: 1,
21015            workflow_signal_id: None,
21016            signal_name: None,
21017            signal_arguments: None,
21018            workflow_update_id: None,
21019            update_name: None,
21020            lease_owner: Some("rust-worker".to_string()),
21021        };
21022
21023        for _redelivery_or_restart in 0..2 {
21024            let commands = worker
21025                .execute_workflow_task(task.clone())
21026                .expect("recorded timer remains pending");
21027            assert!(
21028                commands.is_empty(),
21029                "recorded timer must not be rescheduled"
21030            );
21031        }
21032    }
21033
21034    #[test]
21035    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21036        let client = Client::new("http://127.0.0.1:8080").expect("client");
21037        let mut worker = Worker::new(client, "rust-workers");
21038        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21039            Ok(json!({"status": "completed"}))
21040        });
21041        let task = WorkflowTask {
21042            task_id: "wft-rust-timer-removed".to_string(),
21043            workflow_command_id: None,
21044            workflow_id: Some("wf-rust-timer".to_string()),
21045            run_id: Some("run-rust-timer".to_string()),
21046            workflow_type: "rust.timer.removed".to_string(),
21047            cancel_requested: false,
21048            payload_codec: DEFAULT_CODEC.to_string(),
21049            arguments: Some(
21050                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21051            ),
21052            history_events: vec![
21053                history_event(
21054                    "TimerScheduled",
21055                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21056                ),
21057                history_event(
21058                    "TimerFired",
21059                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21060                ),
21061            ],
21062            total_history_events: Some(2),
21063            history_size_bytes: None,
21064            continue_as_new_recommended: None,
21065            history_budget_pressure: None,
21066            next_history_page_token: None,
21067            workflow_task_attempt: 1,
21068            workflow_signal_id: None,
21069            signal_name: None,
21070            signal_arguments: None,
21071            workflow_update_id: None,
21072            update_name: None,
21073            lease_owner: Some("rust-worker".to_string()),
21074        };
21075
21076        let Error::NonDeterministicReplay(failure) = worker
21077            .execute_workflow_task(task)
21078            .expect_err("removed timer must fail replay")
21079        else {
21080            panic!("expected typed replay failure");
21081        };
21082        assert_eq!(failure.reason, "recorded_commands_unconsumed");
21083        assert_eq!(failure.sequence, Some(1));
21084    }
21085
21086    #[test]
21087    fn workflow_context_emits_explicit_child_workflow_contract() {
21088        let ctx = WorkflowContext {
21089            state: Arc::new(Mutex::new(
21090                WorkflowState::new_with_identity(
21091                    Vec::new(),
21092                    Some("wf-parent".to_string()),
21093                    Some("run-parent".to_string()),
21094                    "parent-workers".to_string(),
21095                    DEFAULT_CODEC.to_string(),
21096                    None,
21097                )
21098                .expect("workflow state"),
21099            )),
21100        };
21101        let options = ChildWorkflowOptions::new("python-workers")
21102            .parent_close_policy(ParentClosePolicy::RequestCancel)
21103            .retry_policy(ChildWorkflowRetryPolicy {
21104                max_attempts: Some(3),
21105                backoff_seconds: vec![1, 5],
21106                non_retryable_error_types: vec!["ValidationError".to_string()],
21107            })
21108            .execution_timeout_seconds(600)
21109            .run_timeout_seconds(120);
21110        let mut call = Box::pin(ctx.start_child_workflow(
21111            "python.fulfil-order",
21112            options,
21113            json!([{"order_id": "order-42"}]),
21114        ));
21115        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21116
21117        assert!(matches!(
21118            call.as_mut().poll(&mut task_context),
21119            Poll::Pending
21120        ));
21121        let commands = ctx.take_commands().expect("commands");
21122        assert_eq!(commands.len(), 1);
21123        let command = &commands[0];
21124        assert_eq!(command["type"], "start_child_workflow");
21125        assert_eq!(command["workflow_type"], "python.fulfil-order");
21126        assert_eq!(command["queue"], "python-workers");
21127        assert_eq!(command["parent_close_policy"], "request_cancel");
21128        assert_eq!(command["retry_policy"]["max_attempts"], 3);
21129        assert_eq!(command["execution_timeout_seconds"], 600);
21130        assert_eq!(command["run_timeout_seconds"], 120);
21131        assert_eq!(
21132            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21133            json!([{"order_id": "order-42"}])
21134        );
21135    }
21136
21137    fn child_parent_worker() -> Worker {
21138        let client = Client::new("http://127.0.0.1:8080").expect("client");
21139        let mut worker = Worker::new(client, "rust-parent-workers");
21140        worker.register_workflow("rust.parent", |ctx, _input| async move {
21141            let child = ctx
21142                .start_child_workflow(
21143                    "python.child",
21144                    ChildWorkflowOptions::new("python-child-workers")
21145                        .parent_close_policy(ParentClosePolicy::Terminate),
21146                    json!([{"codec_probe": [1, true, "rust"]}]),
21147                )
21148                .await?;
21149            Ok(json!({
21150                "parent_workflow_id": child.parent.workflow_id,
21151                "parent_run_id": child.parent.run_id,
21152                "child_workflow_id": child.child.workflow_id,
21153                "child_run_id": child.child.run_id,
21154                "child_workflow_type": child.child_workflow_type,
21155                "result": child.result,
21156            }))
21157        });
21158        worker
21159    }
21160
21161    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21162        WorkflowTask {
21163            task_id: "wft-child-parent".to_string(),
21164            workflow_command_id: None,
21165            workflow_id: Some("wf-parent".to_string()),
21166            run_id: Some("run-parent".to_string()),
21167            workflow_type: "rust.parent".to_string(),
21168            cancel_requested: false,
21169            payload_codec: DEFAULT_CODEC.to_string(),
21170            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21171            history_events: vec![
21172                HistoryEvent {
21173                    event_type: "ChildWorkflowScheduled".to_string(),
21174                    payload: json!({
21175                        "sequence": 1,
21176                        "child_call_id": "call-child",
21177                        "child_workflow_instance_id": "wf-child",
21178                        "child_workflow_run_id": "run-child",
21179                        "child_workflow_type": "python.child",
21180                    }),
21181                    raw: HashMap::new(),
21182                },
21183                HistoryEvent {
21184                    event_type: event_type.to_string(),
21185                    payload,
21186                    raw: HashMap::new(),
21187                },
21188            ],
21189            total_history_events: Some(2),
21190            history_size_bytes: None,
21191            continue_as_new_recommended: None,
21192            history_budget_pressure: None,
21193            next_history_page_token: None,
21194            workflow_task_attempt: 1,
21195            workflow_signal_id: None,
21196            signal_name: None,
21197            signal_arguments: None,
21198            workflow_update_id: None,
21199            update_name: None,
21200            lease_owner: Some("rust-worker".to_string()),
21201        }
21202    }
21203
21204    #[test]
21205    fn committed_child_result_replays_without_starting_a_duplicate() {
21206        let worker = child_parent_worker();
21207        let task = child_parent_task(
21208            "ChildRunCompleted",
21209            json!({
21210                "sequence": 1,
21211                "child_call_id": "call-child",
21212                "child_workflow_instance_id": "wf-child",
21213                "child_workflow_run_id": "run-child",
21214                "child_workflow_type": "python.child",
21215                "payload_codec": DEFAULT_CODEC,
21216                "result": fixture_envelope(json!({"from":"python","ok":true})),
21217            }),
21218        );
21219
21220        for _restart in 0..2 {
21221            let commands = worker
21222                .execute_workflow_task(task.clone())
21223                .expect("replayed parent task");
21224            assert_eq!(commands.len(), 1);
21225            assert_eq!(commands[0]["type"], "complete_workflow");
21226            assert!(!commands
21227                .iter()
21228                .any(|command| command["type"] == "start_child_workflow"));
21229            let output =
21230                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21231            assert_eq!(output["parent_workflow_id"], "wf-parent");
21232            assert_eq!(output["parent_run_id"], "run-parent");
21233            assert_eq!(output["child_workflow_id"], "wf-child");
21234            assert_eq!(output["child_run_id"], "run-child");
21235            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21236        }
21237    }
21238
21239    #[test]
21240    fn typed_child_arguments_and_results_survive_replay() {
21241        let client = Client::new("http://127.0.0.1:8080").expect("client");
21242        let mut worker = Worker::new(client, "rust-parent-workers");
21243        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21244            let child = ctx
21245                .start_child_workflow_avro_value(
21246                    "python.typed-child",
21247                    ChildWorkflowOptions::new("python-workers"),
21248                    AvroValue::Array(vec![typed_fidelity_probe()]),
21249                )
21250                .await?;
21251            Ok(child.result)
21252        });
21253
21254        let initial = worker
21255            .execute_workflow_task(workflow_task(
21256                "rust.typed-parent",
21257                Vec::new(),
21258                DEFAULT_CODEC,
21259            ))
21260            .expect("typed child start");
21261        assert_eq!(initial[0]["type"], "start_child_workflow");
21262        assert_eq!(
21263            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21264                .expect("typed child arguments"),
21265            AvroValue::Array(vec![typed_fidelity_probe()])
21266        );
21267
21268        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21269            .expect("typed child result");
21270        let task = workflow_task(
21271            "rust.typed-parent",
21272            vec![
21273                history_event(
21274                    "ChildWorkflowScheduled",
21275                    json!({
21276                        "sequence": 1,
21277                        "child_call_id": "call-typed",
21278                        "child_workflow_instance_id": "wf-child",
21279                        "child_workflow_run_id": "run-child",
21280                        "child_workflow_type": "python.typed-child",
21281                    }),
21282                ),
21283                history_event(
21284                    "ChildRunCompleted",
21285                    json!({
21286                        "sequence": 1,
21287                        "child_call_id": "call-typed",
21288                        "child_workflow_instance_id": "wf-child",
21289                        "child_workflow_run_id": "run-child",
21290                        "child_workflow_type": "python.typed-child",
21291                        "payload_codec": DEFAULT_CODEC,
21292                        "result": result,
21293                    }),
21294                ),
21295            ],
21296            DEFAULT_CODEC,
21297        );
21298
21299        let commands = worker
21300            .execute_workflow_task(task)
21301            .expect("typed child replay");
21302        assert_eq!(commands[0]["type"], "complete_workflow");
21303        assert_eq!(
21304            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21305                .expect("typed parent result"),
21306            typed_fidelity_probe()
21307        );
21308    }
21309
21310    #[test]
21311    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21312        let worker = child_parent_worker();
21313        let mut task = child_parent_task("unused", Value::Null);
21314        task.history_events.truncate(1);
21315        task.total_history_events = Some(1);
21316
21317        for _redelivery_or_restart in 0..2 {
21318            let commands = worker
21319                .execute_workflow_task(task.clone())
21320                .expect("recorded child remains pending");
21321            assert!(
21322                commands.is_empty(),
21323                "recorded pending child must not be started again"
21324            );
21325        }
21326    }
21327
21328    #[test]
21329    fn child_cancellation_becomes_stable_parent_failure_command() {
21330        let worker = child_parent_worker();
21331        let task = child_parent_task(
21332            "ChildRunCancelled",
21333            json!({
21334                "sequence": 1,
21335                "child_workflow_instance_id": "wf-child",
21336                "child_workflow_run_id": "run-child",
21337                "child_workflow_type": "python.child",
21338                "failure_id": "failure-child",
21339                "failure_category": "cancelled",
21340                "message": "cancelled by parent-close policy",
21341            }),
21342        );
21343
21344        let commands = worker
21345            .execute_workflow_task(task)
21346            .expect("parent settlement");
21347        assert_eq!(commands.len(), 1);
21348        assert_eq!(commands[0]["type"], "fail_workflow");
21349        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21350        assert_eq!(
21351            commands[0]["exception"]["properties"]["reason"],
21352            "cancelled"
21353        );
21354        assert_eq!(
21355            commands[0]["exception"]["properties"]["child_workflow_run_id"],
21356            "run-child"
21357        );
21358    }
21359
21360    #[test]
21361    fn workflow_can_handle_typed_child_failure() {
21362        let client = Client::new("http://127.0.0.1:8080").expect("client");
21363        let mut worker = Worker::new(client, "rust-parent-workers");
21364        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21365            match ctx
21366                .start_child_workflow(
21367                    "python.child",
21368                    ChildWorkflowOptions::new("python-child-workers"),
21369                    json!([]),
21370                )
21371                .await
21372            {
21373                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21374                    "reason": failure.reason,
21375                    "failure_id": failure.failure_id,
21376                    "exception_class": failure.exception_class,
21377                    "child_run_id": failure.child_workflow_run_id,
21378                })),
21379                Err(error) => Err(error),
21380                Ok(_) => Err(Error::WorkerLoop(
21381                    "child unexpectedly succeeded".to_string(),
21382                )),
21383            }
21384        });
21385        let mut task = child_parent_task(
21386            "ChildRunFailed",
21387            json!({
21388                "sequence": 1,
21389                "child_workflow_instance_id": "wf-child",
21390                "child_workflow_run_id": "run-child",
21391                "child_workflow_type": "python.child",
21392                "failure_id": "failure-child",
21393                "failure_category": "child_workflow",
21394                "message": "payment rejected",
21395                "exception": {
21396                    "type": "PaymentRejected",
21397                    "class": "payments.PaymentRejected",
21398                    "message": "payment rejected"
21399                }
21400            }),
21401        );
21402        task.workflow_type = "rust.handled-parent".to_string();
21403
21404        let commands = worker.execute_workflow_task(task).expect("handled failure");
21405        assert_eq!(commands[0]["type"], "complete_workflow");
21406        let output =
21407            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21408        assert_eq!(output["reason"], "child_workflow");
21409        assert_eq!(output["failure_id"], "failure-child");
21410        assert_eq!(output["exception_class"], "payments.PaymentRejected");
21411        assert_eq!(output["child_run_id"], "run-child");
21412    }
21413
21414    #[test]
21415    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21416        let client = Client::new("http://127.0.0.1:8080").expect("client");
21417        let mut worker = Worker::new(client, "rust-workers");
21418
21419        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21420            let signal = ctx.wait_signal("start").await?;
21421            let name = signal
21422                .first()
21423                .and_then(|value| value.as_str())
21424                .unwrap_or("world");
21425            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21426            Ok(json!({
21427                "greeting": greeting,
21428                "language": "rust"
21429            }))
21430        });
21431
21432        let signal_arguments =
21433            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21434        let task = WorkflowTask {
21435            task_id: "wft-rust-signal-1".to_string(),
21436            workflow_command_id: None,
21437            workflow_id: Some("wf-rust-hello".to_string()),
21438            run_id: Some("run-rust-hello".to_string()),
21439            workflow_type: "rust.hello_workflow".to_string(),
21440            cancel_requested: false,
21441            payload_codec: DEFAULT_CODEC.to_string(),
21442            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21443            history_events: vec![HistoryEvent {
21444                event_type: "SignalReceived".to_string(),
21445                payload: json!({
21446                    "signal_id": "sig-rust-1",
21447                    "signal_name": "start"
21448                }),
21449                raw: HashMap::new(),
21450            }],
21451            total_history_events: Some(1),
21452            history_size_bytes: None,
21453            continue_as_new_recommended: None,
21454            history_budget_pressure: None,
21455            next_history_page_token: None,
21456            workflow_task_attempt: 1,
21457            workflow_signal_id: Some("sig-rust-1".to_string()),
21458            signal_name: Some("start".to_string()),
21459            signal_arguments: Some(signal_arguments),
21460            workflow_update_id: None,
21461            update_name: None,
21462            lease_owner: Some("rust-worker".to_string()),
21463        };
21464
21465        let commands = worker.execute_workflow_task(task).expect("workflow task");
21466
21467        assert_eq!(commands.len(), 1);
21468        assert_eq!(commands[0]["type"], "schedule_activity");
21469        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21470        assert_eq!(
21471            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21472            json!(["Rust"])
21473        );
21474    }
21475
21476    #[test]
21477    fn workflow_task_appends_paginated_history_events() {
21478        let mut task = WorkflowTask {
21479            task_id: "wft-rust-pages-1".to_string(),
21480            workflow_command_id: None,
21481            workflow_id: Some("wf-rust-pages".to_string()),
21482            run_id: Some("run-rust-pages".to_string()),
21483            workflow_type: "rust.hello_workflow".to_string(),
21484            cancel_requested: false,
21485            payload_codec: DEFAULT_CODEC.to_string(),
21486            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21487            history_events: vec![HistoryEvent {
21488                event_type: "WorkflowStarted".to_string(),
21489                payload: json!({}),
21490                raw: HashMap::new(),
21491            }],
21492            total_history_events: Some(3),
21493            history_size_bytes: None,
21494            continue_as_new_recommended: None,
21495            history_budget_pressure: None,
21496            next_history_page_token: Some("MQ==".to_string()),
21497            workflow_task_attempt: 1,
21498            workflow_signal_id: None,
21499            signal_name: None,
21500            signal_arguments: None,
21501            workflow_update_id: None,
21502            update_name: None,
21503            lease_owner: Some("rust-worker".to_string()),
21504        };
21505
21506        task.append_history_page(WorkflowTaskHistoryPage {
21507            history_events: vec![
21508                HistoryEvent {
21509                    event_type: "SignalReceived".to_string(),
21510                    payload: json!({
21511                        "signal_id": "sig-rust-1",
21512                        "signal_name": "start",
21513                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21514                            .expect("signal arguments")
21515                    }),
21516                    raw: HashMap::new(),
21517                },
21518                HistoryEvent {
21519                    event_type: "MarkerRecorded".to_string(),
21520                    payload: json!({"sequence": 3}),
21521                    raw: HashMap::new(),
21522                },
21523            ],
21524            total_history_events: Some(3),
21525            next_history_page_token: None,
21526        });
21527
21528        assert_eq!(task.history_events.len(), 3);
21529        assert_eq!(task.total_history_events, Some(3));
21530        assert_eq!(task.next_history_page_token, None);
21531
21532        let signal = task
21533            .history_events
21534            .iter()
21535            .find(|event| event.event_type == "SignalReceived")
21536            .expect("signal event");
21537        assert_eq!(
21538            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21539            vec![AvroValue::String("Rust".to_string())]
21540        );
21541    }
21542
21543    #[tokio::test]
21544    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21545        let client = Client::new("http://127.0.0.1:8080").expect("client");
21546        let mut worker = Worker::new(client, "rust-workers");
21547        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21548        worker.register_query("counter", "current", |ctx, _args| async move {
21549            let mut count = 0_i64;
21550            for signal in ctx.signal_events() {
21551                let value = signal
21552                    .arguments
21553                    .first()
21554                    .and_then(Value::as_i64)
21555                    .unwrap_or_default();
21556                match signal.name.as_str() {
21557                    "increment" => count += value,
21558                    "set" => count = value,
21559                    _ => {}
21560                }
21561            }
21562            Ok(json!(count))
21563        });
21564
21565        let task = QueryTask {
21566            query_task_id: "query-rust-counter".to_string(),
21567            query_task_attempt: 1,
21568            lease_owner: Some("rust-worker".to_string()),
21569            workflow_id: Some("counter-1".to_string()),
21570            run_id: Some("run-counter-1".to_string()),
21571            workflow_type: "counter".to_string(),
21572            query_name: "current".to_string(),
21573            payload_codec: DEFAULT_CODEC.to_string(),
21574            workflow_arguments: Some(
21575                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21576            ),
21577            query_arguments: Some(
21578                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21579            ),
21580            history_events: vec![
21581                HistoryEvent {
21582                    event_type: "SignalReceived".to_string(),
21583                    payload: json!({
21584                        "signal_id": "php-signal-1",
21585                        "signal_name": "increment",
21586                        "workflow_sequence": 1,
21587                        "payload_codec": DEFAULT_CODEC,
21588                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21589                    }),
21590                    raw: HashMap::new(),
21591                },
21592                HistoryEvent {
21593                    event_type: "SignalReceived".to_string(),
21594                    payload: json!({
21595                        "signal_id": "python-signal-2",
21596                        "signal_name": "increment",
21597                        "workflow_sequence": 2,
21598                        "payload_codec": DEFAULT_CODEC,
21599                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21600                    }),
21601                    raw: HashMap::new(),
21602                },
21603                HistoryEvent {
21604                    event_type: "SignalReceived".to_string(),
21605                    payload: json!({
21606                        "signal_id": "rust-signal-3",
21607                        "signal_name": "set",
21608                        "workflow_sequence": 3,
21609                        "payload_codec": DEFAULT_CODEC,
21610                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21611                    }),
21612                    raw: HashMap::new(),
21613                },
21614            ],
21615            history_export: None,
21616            run_status: Some("completed".to_string()),
21617        };
21618
21619        let result = worker.execute_query_task(task).await.expect("query result");
21620        assert_eq!(result.into_json().expect("query projection"), json!(0));
21621    }
21622
21623    #[tokio::test]
21624    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21625        let worker = replay_counter_worker();
21626        let running_history = json!([
21627            {
21628                "type": "ActivityCompleted",
21629                "payload": {
21630                    "sequence": 1,
21631                    "activity_type": "load-counter",
21632                    "payload_codec": DEFAULT_CODEC,
21633                    "result": fixture_envelope(json!("loaded"))
21634                }
21635            },
21636            {
21637                "type": "SignalWaitOpened",
21638                "payload": {
21639                    "sequence": 3,
21640                    "signal_name": "increment"
21641                }
21642            },
21643            {
21644                "type": "SignalReceived",
21645                "payload": {
21646                    "signal_id": "signal-3",
21647                    "signal_name": "increment",
21648                    "workflow_sequence": 2,
21649                    "payload_codec": DEFAULT_CODEC,
21650                    "arguments": fixture_envelope(json!([3]))
21651                }
21652            },
21653            {
21654                "type": "SignalApplied",
21655                "payload": {
21656                    "sequence": 3,
21657                    "signal_id": "signal-3",
21658                    "signal_name": "increment",
21659                    "payload_codec": DEFAULT_CODEC,
21660                    "value": fixture_envelope(json!([3]))
21661                }
21662            }
21663        ]);
21664
21665        let running = worker
21666            .execute_query_task(replay_counter_query(
21667                "current",
21668                running_history.clone(),
21669                "running",
21670            ))
21671            .await
21672            .expect("running replay query");
21673        assert_eq!(
21674            running.clone().into_json().expect("query projection"),
21675            json!({"loaded": "loaded", "count": 3, "finished": false})
21676        );
21677
21678        let detached = worker
21679            .execute_query_task(replay_counter_query(
21680                "detached-mutation",
21681                running_history.clone(),
21682                "running",
21683            ))
21684            .await
21685            .expect("query mutates only its detached state clone");
21686        assert_eq!(detached.into_json().expect("query projection"), json!(999));
21687        let failed = worker
21688            .execute_query_task(replay_counter_query(
21689                "failed-mutation",
21690                running_history.clone(),
21691                "running",
21692            ))
21693            .await
21694            .expect_err("failed query");
21695        assert_eq!(failed.reason, "query_rejected");
21696        let unchanged = worker
21697            .execute_query_task(replay_counter_query("current", running_history, "running"))
21698            .await
21699            .expect("later query reconstructs unchanged state");
21700        assert_eq!(unchanged, running);
21701
21702        let restarted_worker = replay_counter_worker();
21703        let empty_arguments = fixture_envelope(json!([]));
21704        let loaded_result = fixture_envelope(json!("loaded"));
21705        let signal_three = fixture_blob(json!([3]));
21706        let signal_five = fixture_blob(json!([5]));
21707        let restarted_task: QueryTask = serde_json::from_value(json!({
21708            "query_task_id": "query-after-restart",
21709            "workflow_id": "counter-1",
21710            "run_id": "run-counter-1",
21711            "workflow_type": "replay-counter",
21712            "query_name": "current",
21713            "payload_codec": DEFAULT_CODEC,
21714            "workflow_arguments": empty_arguments.clone(),
21715            "query_arguments": empty_arguments,
21716            "history_events": [],
21717            "history_export": {
21718                "payloads": {"codec": DEFAULT_CODEC},
21719                "history_events": [
21720                    {
21721                        "type": "ActivityCompleted",
21722                        "payload": {
21723                            "sequence": 1,
21724                            "activity_type": "load-counter",
21725                            "payload_codec": DEFAULT_CODEC,
21726                            "result": null
21727                        }
21728                    },
21729                    {
21730                        "type": "SignalWaitOpened",
21731                        "payload": {
21732                            "sequence": 3,
21733                            "signal_name": "increment"
21734                        }
21735                    },
21736                    {
21737                        "type": "SignalReceived",
21738                        "payload": {
21739                            "signal_id": "signal-3",
21740                            "signal_name": "increment",
21741                            "workflow_sequence": 2
21742                        }
21743                    },
21744                    {
21745                        "type": "SignalApplied",
21746                        "payload": {
21747                            "sequence": 3,
21748                            "signal_id": "signal-3",
21749                            "signal_name": "increment"
21750                        }
21751                    },
21752                    {
21753                        "type": "SignalWaitOpened",
21754                        "payload": {
21755                            "sequence": 5,
21756                            "signal_name": "increment"
21757                        }
21758                    },
21759                    {
21760                        "type": "SignalReceived",
21761                        "payload": {
21762                            "signal_id": "signal-5",
21763                            "signal_name": "increment",
21764                            "workflow_sequence": 4
21765                        }
21766                    },
21767                    {
21768                        "type": "SignalApplied",
21769                        "payload": {
21770                            "sequence": 5,
21771                            "signal_id": "signal-5",
21772                            "signal_name": "increment"
21773                        }
21774                    }
21775                ],
21776                "activities": [{
21777                    "sequence": 1,
21778                    "activity_type": "load-counter",
21779                    "payload_codec": DEFAULT_CODEC,
21780                    "result": loaded_result
21781                }],
21782                "signals": [
21783                    {
21784                        "id": "signal-3",
21785                        "name": "increment",
21786                        "workflow_sequence": 2,
21787                        "payload_codec": DEFAULT_CODEC,
21788                        "arguments": signal_three
21789                    },
21790                    {
21791                        "id": "signal-5",
21792                        "name": "increment",
21793                        "workflow_sequence": 4,
21794                        "payload_codec": DEFAULT_CODEC,
21795                        "arguments": signal_five
21796                    }
21797                ]
21798            },
21799            "run_status": "completed"
21800        }))
21801        .expect("cold replay query task");
21802        let completed = restarted_worker
21803            .execute_query_task(restarted_task)
21804            .await
21805            .expect("completed cold replay query");
21806        assert_eq!(
21807            completed.into_json().expect("query projection"),
21808            json!({"loaded": "loaded", "count": 8, "finished": true})
21809        );
21810    }
21811
21812    #[tokio::test]
21813    async fn replayed_query_replay_failures_are_machine_readable() {
21814        let worker = replay_counter_worker();
21815        let task = replay_counter_query(
21816            "current",
21817            json!([{
21818                "type": "ActivityCompleted",
21819                "payload": {
21820                    "sequence": 1,
21821                    "payload_codec": DEFAULT_CODEC,
21822                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21823                }
21824            }]),
21825            "running",
21826        );
21827        let failure = worker
21828            .execute_query_task(task)
21829            .await
21830            .expect_err("invalid replay history payload");
21831        assert_eq!(failure.reason, "query_payload_decode_failed");
21832        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21833        assert!(failure.message.contains("invalid_payload_framing"));
21834    }
21835
21836    #[tokio::test]
21837    async fn query_task_restores_compact_history_from_export() {
21838        let client = Client::new("http://127.0.0.1:8080").expect("client");
21839        let mut worker = Worker::new(client, "rust-workers");
21840        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21841        worker.register_query("counter", "current", |ctx, _args| async move {
21842            Ok(json!(ctx.signals("increment")[0][0]))
21843        });
21844        let empty_arguments = fixture_envelope(json!([]));
21845        let exported_signal = fixture_blob(json!([9]));
21846        let task: QueryTask = serde_json::from_value(json!({
21847            "query_task_id": "query-export",
21848            "workflow_type": "counter",
21849            "query_name": "current",
21850            "payload_codec": DEFAULT_CODEC,
21851            "workflow_arguments": empty_arguments.clone(),
21852            "query_arguments": empty_arguments,
21853            "history_events": [],
21854            "history_export": {
21855                "payloads": {"codec": DEFAULT_CODEC},
21856                "history_events": [{
21857                    "type": "SignalReceived",
21858                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21859                }],
21860                "signals": [{
21861                    "id": "signal-export",
21862                    "name": "increment",
21863                    "status": "applied",
21864                    "workflow_sequence": 1,
21865                    "payload_codec": DEFAULT_CODEC,
21866                    "arguments": exported_signal
21867                }]
21868            }
21869        }))
21870        .expect("query task");
21871
21872        let result = worker.execute_query_task(task).await.expect("query result");
21873        assert_eq!(result.into_json().expect("query projection"), json!(9));
21874    }
21875
21876    #[tokio::test]
21877    async fn query_task_failures_have_stable_reasons() {
21878        let client = Client::new("http://127.0.0.1:8080").expect("client");
21879        let mut worker = Worker::new(client, "rust-workers");
21880        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21881        worker.register_query(
21882            "counter",
21883            "current",
21884            |_ctx, _args| async move { Ok(json!(0)) },
21885        );
21886
21887        let base_task = QueryTask {
21888            query_task_id: "query-errors".to_string(),
21889            query_task_attempt: 1,
21890            lease_owner: None,
21891            workflow_id: Some("counter-errors".to_string()),
21892            run_id: Some("run-errors".to_string()),
21893            workflow_type: "counter".to_string(),
21894            query_name: "missing".to_string(),
21895            payload_codec: DEFAULT_CODEC.to_string(),
21896            workflow_arguments: Some(fixture_envelope(json!([]))),
21897            query_arguments: Some(fixture_envelope(json!([]))),
21898            history_events: Vec::new(),
21899            history_export: None,
21900            run_status: Some("running".to_string()),
21901        };
21902
21903        let unknown = worker
21904            .execute_query_task(base_task.clone())
21905            .await
21906            .expect_err("unknown query");
21907        assert_eq!(unknown.reason, "rejected_unknown_query");
21908
21909        let mut malformed = base_task;
21910        malformed.query_name = "current".to_string();
21911        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
21912        let malformed = worker
21913            .execute_query_task(malformed)
21914            .await
21915            .expect_err("malformed payload");
21916        assert_eq!(malformed.reason, "query_payload_decode_failed");
21917
21918        let client = Client::new("http://127.0.0.1:8080").expect("client");
21919        let mut unavailable_worker = Worker::new(client, "rust-workers");
21920        unavailable_worker
21921            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21922        let empty_arguments = fixture_envelope(json!([]));
21923        let unavailable_task: QueryTask = serde_json::from_value(json!({
21924            "query_task_id": "query-unavailable",
21925            "workflow_type": "counter",
21926            "query_name": "current",
21927            "payload_codec": DEFAULT_CODEC,
21928            "workflow_arguments": empty_arguments.clone(),
21929            "query_arguments": empty_arguments
21930        }))
21931        .expect("query task");
21932        let unavailable = unavailable_worker
21933            .execute_query_task(unavailable_task)
21934            .await
21935            .expect_err("query handler unavailable");
21936        assert_eq!(unavailable.reason, "query_handler_unavailable");
21937    }
21938
21939    #[tokio::test]
21940    async fn client_query_decodes_result_and_typed_failure() {
21941        let server = MockWorkerServer::start();
21942        let client = Client::builder(server.base_url())
21943            .timeout(Duration::from_secs(2))
21944            .build()
21945            .expect("client");
21946
21947        let result = client
21948            .query_workflow("counter-1", "current", json!([]))
21949            .await
21950            .expect("query result");
21951        assert_eq!(result, json!({"count": 8}));
21952
21953        let error = client
21954            .query_workflow("counter-1", "missing", json!([]))
21955            .await
21956            .expect_err("unknown query");
21957        let Error::QueryFailed(failure) = error else {
21958            panic!("expected typed query failure");
21959        };
21960        assert_eq!(failure.status, 404);
21961        assert_eq!(failure.reason, "rejected_unknown_query");
21962    }
21963
21964    #[tokio::test]
21965    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
21966        let server = MockWorkerServer::start();
21967        let client = Client::builder(server.base_url())
21968            .timeout(Duration::from_secs(2))
21969            .build()
21970            .expect("client");
21971        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
21972
21973        client
21974            .start_workflow(
21975                "typed.echo",
21976                "rust-workers",
21977                "typed-start",
21978                arguments.clone(),
21979            )
21980            .await
21981            .expect("typed workflow start");
21982        assert_eq!(
21983            decode_wire_avro_value(
21984                &server.request_body("/api/workflows")["input"],
21985                DEFAULT_CODEC,
21986            )
21987            .expect("typed start input"),
21988            arguments
21989        );
21990
21991        client
21992            .signal_workflow("typed-1", "changed", arguments.clone())
21993            .await
21994            .expect("typed signal");
21995        assert_eq!(
21996            decode_wire_avro_value(
21997                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
21998                DEFAULT_CODEC,
21999            )
22000            .expect("typed signal input"),
22001            arguments
22002        );
22003
22004        assert_eq!(
22005            client
22006                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22007                .await
22008                .expect("typed query"),
22009            typed_fidelity_probe()
22010        );
22011        assert_eq!(
22012            decode_wire_avro_value(
22013                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22014                DEFAULT_CODEC,
22015            )
22016            .expect("typed query input"),
22017            arguments
22018        );
22019
22020        assert_eq!(
22021            client
22022                .update_workflow_avro_value(
22023                    "typed-1",
22024                    "replace",
22025                    arguments.clone(),
22026                    Some("typed-request"),
22027                )
22028                .await
22029                .expect("typed update"),
22030            typed_fidelity_probe()
22031        );
22032        let update = server.request_body("/api/workflows/typed-1/update/replace");
22033        assert_eq!(update["request_id"], "typed-request");
22034        assert_eq!(
22035            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22036            arguments
22037        );
22038
22039        let handle = WorkflowHandle {
22040            client: client.clone(),
22041            workflow_id: "typed-1".to_string(),
22042            run_id: Some("run-typed-1".to_string()),
22043            workflow_type: "typed.echo".to_string(),
22044        };
22045        assert_eq!(
22046            handle
22047                .result_avro_value(WorkflowResultOptions::default())
22048                .await
22049                .expect("typed workflow result"),
22050            typed_fidelity_probe()
22051        );
22052
22053        client
22054            .complete_activity_task(
22055                "activity-typed",
22056                "attempt-typed",
22057                "rust-worker",
22058                typed_fidelity_probe(),
22059                DEFAULT_CODEC,
22060            )
22061            .await
22062            .expect("typed activity completion");
22063        assert_eq!(
22064            decode_wire_avro_value(
22065                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22066                    ["result"],
22067                DEFAULT_CODEC,
22068            )
22069            .expect("typed activity result"),
22070            typed_fidelity_probe()
22071        );
22072        client
22073            .fail_activity_task(
22074                "activity-typed",
22075                "attempt-typed",
22076                "rust-worker",
22077                "typed failure",
22078                true,
22079            )
22080            .await
22081            .expect("activity failure");
22082    }
22083
22084    #[tokio::test]
22085    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22086        let server = MockWorkerServer::start();
22087        let client = Client::builder(server.base_url())
22088            .timeout(Duration::from_secs(2))
22089            .build()
22090            .expect("client");
22091
22092        let options = WorkflowCommandOptions::new()
22093            .reason("cleanup requested")
22094            .request_id("cancel-17");
22095        let cancelled = client
22096            .cancel_workflow("wf-lifecycle", options)
22097            .await
22098            .expect("instance cancellation");
22099        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22100        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22101        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22102        assert_eq!(
22103            server.request_body("/api/workflows/wf-lifecycle/cancel"),
22104            json!({"reason":"cleanup requested","request_id":"cancel-17"})
22105        );
22106
22107        let terminated = client
22108            .terminate_workflow(
22109                "wf-lifecycle",
22110                WorkflowCommandOptions::new().reason("forced stop"),
22111            )
22112            .await
22113            .expect("instance termination");
22114        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22115        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22116
22117        client
22118            .cancel_workflow_run(
22119                "wf-lifecycle",
22120                "run-current",
22121                WorkflowCommandOptions::default(),
22122            )
22123            .await
22124            .expect("selected run cancellation");
22125        client
22126            .terminate_workflow_run(
22127                "wf-lifecycle",
22128                "run-current",
22129                WorkflowCommandOptions::default(),
22130            )
22131            .await
22132            .expect("selected run termination");
22133
22134        for (command, error) in [
22135            (
22136                WorkflowCommandKind::Cancel,
22137                client
22138                    .cancel_workflow_run(
22139                        "wf-lifecycle",
22140                        "run-stale",
22141                        WorkflowCommandOptions::default(),
22142                    )
22143                    .await
22144                    .expect_err("stale cancellation must be rejected"),
22145            ),
22146            (
22147                WorkflowCommandKind::Terminate,
22148                client
22149                    .terminate_workflow_run(
22150                        "wf-lifecycle",
22151                        "run-stale",
22152                        WorkflowCommandOptions::default(),
22153                    )
22154                    .await
22155                    .expect_err("stale termination must be rejected"),
22156            ),
22157        ] {
22158            let Error::WorkflowCommandRejected(rejection) = error else {
22159                panic!("expected typed command rejection");
22160            };
22161            assert_eq!(rejection.command, command);
22162            assert_eq!(rejection.status, 409);
22163            assert_eq!(rejection.reason, "historical_run_command_rejected");
22164            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22165            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22166        }
22167    }
22168
22169    #[tokio::test]
22170    async fn workflow_start_options_send_server_enforced_deadlines() {
22171        let server = MockWorkerServer::start();
22172        let client = Client::builder(server.base_url())
22173            .timeout(Duration::from_secs(2))
22174            .build()
22175            .expect("client");
22176
22177        let handle = client
22178            .start_workflow_with_options(
22179                "rust.timeout",
22180                "rust-timeouts",
22181                "wf-start-options",
22182                WorkflowStartOptions::new()
22183                    .execution_timeout_seconds(30)
22184                    .run_timeout_seconds(1),
22185                json!([]),
22186            )
22187            .await
22188            .expect("workflow start");
22189
22190        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22191        let body = server.request_body("/api/workflows");
22192        assert_eq!(body["execution_timeout_seconds"], 30);
22193        assert_eq!(body["run_timeout_seconds"], 1);
22194
22195        let invalid = client
22196            .start_workflow_with_options(
22197                "rust.timeout",
22198                "rust-timeouts",
22199                "wf-invalid-options",
22200                WorkflowStartOptions::new()
22201                    .execution_timeout_seconds(1)
22202                    .run_timeout_seconds(2),
22203                json!([]),
22204            )
22205            .await
22206            .expect_err("invalid deadline ordering");
22207        assert!(invalid
22208            .to_string()
22209            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22210    }
22211
22212    #[tokio::test]
22213    async fn workflow_result_returns_each_typed_terminal_outcome() {
22214        let server = MockWorkerServer::start();
22215        let client = Client::builder(server.base_url())
22216            .timeout(Duration::from_secs(2))
22217            .build()
22218            .expect("client");
22219        let options = WorkflowResultOptions {
22220            poll_interval: Duration::ZERO,
22221            timeout: Duration::from_secs(1),
22222        };
22223
22224        let failed = WorkflowHandle {
22225            client: client.clone(),
22226            workflow_id: "wf-failed".to_string(),
22227            run_id: Some("run-failed".to_string()),
22228            workflow_type: "failure".to_string(),
22229        }
22230        .result(options)
22231        .await
22232        .expect_err("failed outcome");
22233        let Error::WorkflowFailed(failure) = failed else {
22234            panic!("expected WorkflowFailed");
22235        };
22236        assert_eq!(failure.workflow_id, "wf-failed");
22237        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22238        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22239        assert_eq!(failure.failure_category.as_deref(), Some("application"));
22240        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22241        assert_eq!(
22242            failure.exception_class.as_deref(),
22243            Some("billing::PaymentError")
22244        );
22245        assert_eq!(failure.non_retryable, Some(true));
22246
22247        for (workflow_id, expected_kind, expected_reason) in [
22248            (
22249                "wf-cancelled",
22250                WorkflowTerminalKind::Cancelled,
22251                "cleanup requested",
22252            ),
22253            (
22254                "wf-terminated",
22255                WorkflowTerminalKind::Terminated,
22256                "forced stop",
22257            ),
22258            (
22259                "wf-timed-out",
22260                WorkflowTerminalKind::TimedOut,
22261                "run_timeout",
22262            ),
22263        ] {
22264            let error = WorkflowHandle {
22265                client: client.clone(),
22266                workflow_id: workflow_id.to_string(),
22267                run_id: None,
22268                workflow_type: "terminal".to_string(),
22269            }
22270            .result(options)
22271            .await
22272            .expect_err("typed terminal outcome");
22273            let outcome = match error {
22274                Error::WorkflowCancelled(outcome) => outcome,
22275                Error::WorkflowTerminated(outcome) => outcome,
22276                Error::WorkflowTimedOut(outcome) => outcome,
22277                other => panic!("unexpected terminal error: {other}"),
22278            };
22279            assert_eq!(outcome.kind, expected_kind);
22280            assert_eq!(outcome.workflow_id, workflow_id);
22281            assert_eq!(outcome.reason, expected_reason);
22282        }
22283
22284        let wait_timeout = WorkflowHandle {
22285            client,
22286            workflow_id: "wf-waiting".to_string(),
22287            run_id: Some("run-waiting".to_string()),
22288            workflow_type: "waiting".to_string(),
22289        }
22290        .result(WorkflowResultOptions {
22291            poll_interval: Duration::ZERO,
22292            timeout: Duration::ZERO,
22293        })
22294        .await
22295        .expect_err("client wait timeout");
22296        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22297            panic!("expected typed client timeout");
22298        };
22299        assert_eq!(timeout.reason, "result_wait_timeout");
22300        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22301        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22302    }
22303
22304    #[tokio::test]
22305    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22306        let server = MockWorkerServer::start();
22307        let client = Client::builder(server.base_url())
22308            .timeout(Duration::from_secs(2))
22309            .build()
22310            .expect("client");
22311
22312        let handle = WorkflowHandle {
22313            client,
22314            workflow_id: "wf-selected".to_string(),
22315            run_id: Some("run-selected".to_string()),
22316            workflow_type: "selected".to_string(),
22317        };
22318        let options = WorkflowResultOptions {
22319            poll_interval: Duration::ZERO,
22320            timeout: Duration::from_secs(1),
22321        };
22322
22323        let current = handle
22324            .result(options)
22325            .await
22326            .expect("instance result follows the current run");
22327        assert_eq!(current, json!("current run output"));
22328
22329        let error = handle
22330            .result_selected_run(options)
22331            .await
22332            .expect_err("the selected run is cancelled even though the current run completed");
22333
22334        let Error::WorkflowCancelled(outcome) = error else {
22335            panic!("expected selected run cancellation");
22336        };
22337        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22338        assert_eq!(outcome.reason, "selected run cancelled");
22339        assert_eq!(
22340            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22341            1
22342        );
22343        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22344    }
22345
22346    #[tokio::test]
22347    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22348        let server = MockWorkerServer::draining_polls();
22349        let client = Client::builder(server.base_url())
22350            .timeout(Duration::from_secs(2))
22351            .build()
22352            .expect("client");
22353
22354        let workflow = client
22355            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22356            .await
22357            .expect("workflow drain response");
22358        let activity = client
22359            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22360            .await
22361            .expect("activity drain response");
22362        let query = client
22363            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22364            .await
22365            .expect("query drain response");
22366
22367        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22368            assert_eq!(
22369                outcome,
22370                WorkerPollOutcome::Stop {
22371                    poll_status: Some("draining".to_string()),
22372                    reason: Some("worker_draining".to_string()),
22373                }
22374            );
22375        }
22376
22377        assert!(client
22378            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22379            .await
22380            .expect("compatibility poll")
22381            .is_none());
22382    }
22383
22384    #[tokio::test]
22385    async fn managed_worker_honors_drain_stop_for_every_task_family() {
22386        let server = MockWorkerServer::draining_polls();
22387        let client = Client::builder(server.base_url())
22388            .timeout(Duration::from_secs(2))
22389            .build()
22390            .expect("client");
22391
22392        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22393            .worker_id("draining-workflow-worker")
22394            .poll_timeout(Duration::ZERO);
22395        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22396        workflow_worker
22397            .run()
22398            .await
22399            .expect("workflow drain is a clean stop");
22400
22401        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22402            .worker_id("draining-activity-worker")
22403            .poll_timeout(Duration::ZERO);
22404        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22405        activity_worker
22406            .run()
22407            .await
22408            .expect("activity drain is a clean stop");
22409
22410        let mut query_worker = Worker::new(client, "rust-workers")
22411            .worker_id("draining-query-worker")
22412            .poll_timeout(Duration::ZERO);
22413        query_worker.register_query("counter", "current", |_ctx, _args| async {
22414            Ok(Value::Null)
22415        });
22416        query_worker
22417            .run()
22418            .await
22419            .expect("query drain is a clean stop");
22420    }
22421
22422    #[tokio::test]
22423    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22424        let server = MockWorkerServer::start();
22425        let client = Client::builder(server.base_url())
22426            .timeout(Duration::from_secs(2))
22427            .build()
22428            .expect("client");
22429
22430        let heartbeat = client
22431            .heartbeat_activity_task(
22432                "activity-cancel",
22433                "attempt-cancel",
22434                "rust-worker",
22435                typed_fidelity_probe(),
22436            )
22437            .await
22438            .expect("cancellation heartbeat");
22439        assert!(heartbeat.cancel_requested);
22440        assert!(heartbeat.should_stop());
22441        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22442        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22443        let heartbeat_body =
22444            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22445        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22446        assert_eq!(
22447            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22448                .expect("typed heartbeat details"),
22449            typed_fidelity_probe()
22450        );
22451
22452        let error = client
22453            .complete_activity_task(
22454                "activity-cancel",
22455                "attempt-cancel",
22456                "rust-worker",
22457                json!({"late":true}),
22458                DEFAULT_CODEC,
22459            )
22460            .await
22461            .expect_err("late completion must be refused");
22462        assert!(activity_task_rejection_is_final(&error));
22463        let Error::ActivityTaskRejected(rejection) = error else {
22464            panic!("expected typed activity rejection");
22465        };
22466        assert_eq!(rejection.status, 409);
22467        assert_eq!(rejection.reason, "run_cancelled");
22468        assert!(rejection.cancel_requested);
22469        assert_eq!(rejection.can_continue, Some(false));
22470    }
22471
22472    #[tokio::test]
22473    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22474        let server = MockWorkerServer::cancelled_activity();
22475        let client = Client::builder(server.base_url())
22476            .timeout(Duration::from_secs(2))
22477            .build()
22478            .expect("client");
22479        let cancellation_observed = Arc::new(AtomicBool::new(false));
22480        let observed = Arc::clone(&cancellation_observed);
22481        let mut worker = Worker::new(client.clone(), "rust-workers")
22482            .worker_id("rust-cancel-worker")
22483            .poll_timeout(Duration::from_millis(10));
22484        worker.register_activity("cancel-aware", move |ctx, _args| {
22485            let observed = Arc::clone(&observed);
22486            async move {
22487                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22488                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22489                Ok(json!({"late":"completion"}))
22490            }
22491        });
22492
22493        assert_eq!(
22494            worker.run_once().await.expect("cancelled attempt handled"),
22495            1
22496        );
22497        assert!(cancellation_observed.load(Ordering::SeqCst));
22498        assert_eq!(
22499            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22500            1
22501        );
22502
22503        let mut restarted = Worker::new(client, "rust-workers")
22504            .worker_id("rust-cancel-worker-restarted")
22505            .poll_timeout(Duration::from_millis(10));
22506        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22507        assert_eq!(
22508            restarted
22509                .run_once()
22510                .await
22511                .expect("replacement worker continues polling"),
22512            0
22513        );
22514    }
22515
22516    #[tokio::test]
22517    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22518        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22519        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22520        let client = Client::builder(server.base_url())
22521            .timeout(Duration::from_secs(2))
22522            .build()
22523            .expect("client");
22524
22525        let direct_error = client
22526            .complete_workflow_task(
22527                "workflow-timeout-task",
22528                "timeout-worker",
22529                3,
22530                vec![json!({
22531                    "type": "complete_workflow",
22532                    "result": fixture_envelope(Value::Null)
22533                })],
22534            )
22535            .await
22536            .expect_err("the low-level client preserves the completion rejection");
22537        let Error::Http { status, body } = direct_error else {
22538            panic!("expected the original HTTP completion rejection");
22539        };
22540        assert_eq!(status, reqwest::StatusCode::CONFLICT);
22541        assert_eq!(
22542            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22543            "run_timed_out"
22544        );
22545
22546        let mut worker = Worker::new(client, "rust-workers")
22547            .worker_id("timeout-worker")
22548            .poll_timeout(Duration::from_millis(10));
22549        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22550            Ok(json!({"late": "result"}))
22551        });
22552
22553        assert_eq!(
22554            worker
22555                .run_once()
22556                .await
22557                .expect("authoritative selected-run timeout settles the tick"),
22558            1
22559        );
22560        assert_eq!(
22561            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22562            2,
22563            "both the direct client proof and managed worker must see the rejection"
22564        );
22565    }
22566
22567    #[tokio::test]
22568    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22569        for (name, status, response) in [
22570            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22571            (
22572                "command was recorded",
22573                "409 Conflict",
22574                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22575            ),
22576            (
22577                "lease conflict",
22578                "409 Conflict",
22579                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22580            ),
22581            (
22582                "nonterminal run",
22583                "409 Conflict",
22584                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22585            ),
22586            (
22587                "different selected run",
22588                "409 Conflict",
22589                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22590            ),
22591            (
22592                "different task attempt",
22593                "409 Conflict",
22594                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22595            ),
22596            (
22597                "authentication failure",
22598                "401 Unauthorized",
22599                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22600            ),
22601            (
22602                "authorization failure",
22603                "403 Forbidden",
22604                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22605            ),
22606            (
22607                "protocol failure",
22608                "400 Bad Request",
22609                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22610            ),
22611            (
22612                "malformed command",
22613                "422 Unprocessable Entity",
22614                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22615            ),
22616            (
22617                "transient server failure",
22618                "503 Service Unavailable",
22619                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22620            ),
22621        ] {
22622            let server = MockWorkerServer::workflow_completion(status, response);
22623            let client = Client::builder(server.base_url())
22624                .timeout(Duration::from_secs(2))
22625                .build()
22626                .expect("client");
22627            let mut worker = Worker::new(client, "rust-workers")
22628                .worker_id("timeout-worker")
22629                .poll_timeout(Duration::from_millis(10));
22630            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22631                Ok(json!({"late": "result"}))
22632            });
22633
22634            let error = worker
22635                .run_once()
22636                .await
22637                .expect_err(&format!("{name} must remain an error"));
22638            assert!(
22639                matches!(error, Error::Http { .. } | Error::Protocol(_)),
22640                "{name} returned an unexpected error variant: {error}"
22641            );
22642        }
22643    }
22644
22645    #[tokio::test]
22646    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22647        let server = MockWorkerServer::start();
22648        let client = Client::builder(server.base_url())
22649            .worker_token(Some("worker-secret".to_string()))
22650            .namespace("orders")
22651            .timeout(Duration::from_secs(2))
22652            .build()
22653            .expect("client");
22654        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22655
22656        let result = client
22657            .deregister_worker_registration("worker/α space")
22658            .await
22659            .expect("deregister worker registration");
22660
22661        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22662        assert_eq!(
22663            server.worker_protocol_for(path).as_deref(),
22664            Some(WORKER_PROTOCOL_VERSION)
22665        );
22666        assert_eq!(server.control_protocol_for(path), None);
22667        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22668        assert_eq!(
22669            server.authorization_for(path).as_deref(),
22670            Some("Bearer worker-secret")
22671        );
22672        assert_eq!(
22673            result,
22674            WorkerDeregistrationEnvelope {
22675                worker_id: "deregistered-worker".to_string(),
22676                outcome: "deregistered".to_string(),
22677                recovered_workflow_task_count: 2,
22678            }
22679        );
22680    }
22681
22682    #[tokio::test]
22683    async fn low_level_registration_rejects_update_validators_before_transport() {
22684        let server = MockWorkerServer::start();
22685        let client = Client::builder(server.base_url())
22686            .timeout(Duration::from_secs(2))
22687            .build()
22688            .expect("client");
22689
22690        for update_validators in [json!(["approve"]), json!("approve")] {
22691            let error = client
22692                .register_worker_with_command_contracts(
22693                    "validator-claiming-worker",
22694                    "rust-workers",
22695                    vec!["orders".to_string()],
22696                    vec![],
22697                    1,
22698                    1,
22699                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22700                    json!({
22701                        "orders": {
22702                            "queries": ["current"],
22703                            "updates": ["approve"],
22704                            "update_validators": update_validators,
22705                        },
22706                    }),
22707                )
22708                .await
22709                .expect_err("unsupported validator claims must fail before registration");
22710
22711            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22712                panic!("expected typed unsupported-validator failure");
22713            };
22714            assert_eq!(workflow_type, "orders");
22715        }
22716        assert_eq!(server.request_count("/api/worker/register"), 0);
22717    }
22718
22719    #[tokio::test]
22720    async fn low_level_registration_preserves_query_and_update_contracts() {
22721        let server = MockWorkerServer::start();
22722        let client = Client::builder(server.base_url())
22723            .timeout(Duration::from_secs(2))
22724            .build()
22725            .expect("client");
22726        let contracts = json!({
22727            "orders": {
22728                "queries": ["current"],
22729                "updates": ["approve"],
22730                "update_validators": [],
22731            },
22732            "payments": {
22733                "queries": ["status"],
22734                "updates": ["capture"],
22735            },
22736        });
22737
22738        client
22739            .register_worker_with_command_contracts(
22740                "command-worker",
22741                "rust-workers",
22742                vec!["orders".to_string(), "payments".to_string()],
22743                vec![],
22744                1,
22745                1,
22746                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22747                contracts.clone(),
22748            )
22749            .await
22750            .expect("query and update contracts must remain supported");
22751
22752        assert_eq!(
22753            server.request_body("/api/worker/register")["workflow_command_contracts"],
22754            contracts
22755        );
22756    }
22757
22758    #[tokio::test]
22759    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22760        let server = MockWorkerServer::start();
22761        let control_only = Client::builder(server.base_url())
22762            .control_token(Some("control-secret".to_string()))
22763            .build()
22764            .expect("control client");
22765
22766        let error = control_only
22767            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22768            .await
22769            .expect_err("control token must not authorize a worker request");
22770        assert!(matches!(
22771            error,
22772            Error::MissingRoleCredentials { role: "worker", .. }
22773        ));
22774        assert_eq!(server.request_count("/api/worker/register"), 0);
22775
22776        let worker_only = Client::builder(server.base_url())
22777            .worker_token(Some("worker-secret".to_string()))
22778            .build()
22779            .expect("worker client");
22780        let error = worker_only
22781            .health()
22782            .await
22783            .expect_err("worker token must not authorize a control request");
22784        assert!(matches!(
22785            error,
22786            Error::MissingRoleCredentials {
22787                role: "control",
22788                ..
22789            }
22790        ));
22791        assert_eq!(server.request_count("/api/health"), 0);
22792    }
22793
22794    #[tokio::test]
22795    async fn shared_token_supports_worker_and_control_planes() {
22796        let server = MockWorkerServer::start();
22797        let client = Client::builder(server.base_url())
22798            .token(Some("shared-secret".to_string()))
22799            .build()
22800            .expect("client");
22801
22802        client.health().await.expect("control request");
22803        client
22804            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22805            .await
22806            .expect("worker request");
22807
22808        assert_eq!(
22809            server.authorization_for("/api/health").as_deref(),
22810            Some("Bearer shared-secret")
22811        );
22812        assert_eq!(
22813            server.control_protocol_for("/api/health").as_deref(),
22814            Some(CONTROL_PLANE_VERSION)
22815        );
22816        assert_eq!(
22817            server.authorization_for("/api/worker/register").as_deref(),
22818            Some("Bearer shared-secret")
22819        );
22820        assert_eq!(
22821            server
22822                .worker_protocol_for("/api/worker/register")
22823                .as_deref(),
22824            Some(WORKER_PROTOCOL_VERSION)
22825        );
22826    }
22827
22828    #[tokio::test]
22829    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22830        let server = MockWorkerServer::start();
22831        let client = Client::builder(server.base_url())
22832            .timeout(Duration::from_secs(2))
22833            .build()
22834            .expect("client");
22835
22836        client
22837            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22838            .await
22839            .expect("register");
22840        client
22841            .heartbeat_worker("capture-worker", 1, 1)
22842            .await
22843            .expect("heartbeat");
22844        client
22845            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22846            .await
22847            .expect("workflow poll");
22848        client
22849            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22850            .await
22851            .expect("activity poll");
22852
22853        for path in [
22854            "/api/worker/register",
22855            "/api/worker/heartbeat",
22856            "/api/worker/workflow-tasks/poll",
22857            "/api/worker/activity-tasks/poll",
22858        ] {
22859            assert_eq!(
22860                server.worker_protocol_for(path).as_deref(),
22861                Some(WORKER_PROTOCOL_VERSION),
22862                "unexpected protocol for {path}"
22863            );
22864        }
22865
22866        assert_eq!(
22867            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22868            1
22869        );
22870        assert_eq!(
22871            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22872            1
22873        );
22874        assert!(
22875            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22876                .as_str()
22877                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22878        );
22879        assert!(
22880            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22881                .as_str()
22882                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22883        );
22884    }
22885
22886    #[tokio::test]
22887    async fn query_task_endpoints_send_the_query_feature_protocol() {
22888        let server = MockWorkerServer::start();
22889        let client = Client::builder(server.base_url())
22890            .timeout(Duration::from_secs(2))
22891            .build()
22892            .expect("client");
22893
22894        client
22895            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22896            .await
22897            .expect("query poll");
22898        client
22899            .complete_query_task(
22900                "query-capture",
22901                "capture-worker",
22902                1,
22903                json!(8),
22904                DEFAULT_CODEC,
22905            )
22906            .await
22907            .expect("query complete");
22908        client
22909            .fail_query_task(
22910                "query-capture",
22911                "capture-worker",
22912                1,
22913                "failed",
22914                "query_rejected",
22915                "QueryFailed",
22916            )
22917            .await
22918            .expect("query fail");
22919
22920        for path in [
22921            "/api/worker/query-tasks/poll",
22922            "/api/worker/query-tasks/query-capture/complete",
22923            "/api/worker/query-tasks/query-capture/fail",
22924        ] {
22925            assert_eq!(
22926                server.worker_protocol_for(path).as_deref(),
22927                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
22928                "unexpected protocol for {path}"
22929            );
22930        }
22931
22932        assert_eq!(
22933            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
22934            1
22935        );
22936        assert!(
22937            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
22938                .as_str()
22939                .is_some_and(|id| id.starts_with("rust-query-poll-"))
22940        );
22941    }
22942
22943    #[tokio::test]
22944    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
22945        let server = MockWorkerServer::transient_worker_failures();
22946        let client = Client::builder(server.base_url())
22947            .timeout(Duration::from_secs(2))
22948            .build()
22949            .expect("client");
22950
22951        client
22952            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22953            .await
22954            .expect("workflow poll retry");
22955        client
22956            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22957            .await
22958            .expect("activity poll retry");
22959        client
22960            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22961            .await
22962            .expect("query poll retry");
22963
22964        for path in [
22965            "/api/worker/workflow-tasks/poll",
22966            "/api/worker/activity-tasks/poll",
22967            "/api/worker/query-tasks/poll",
22968        ] {
22969            let bodies = server.request_bodies(path);
22970            assert_eq!(bodies.len(), 2, "{path} must be retried once");
22971            assert_eq!(
22972                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
22973                "{path} must preserve the request binding across retry"
22974            );
22975        }
22976    }
22977
22978    #[tokio::test]
22979    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
22980        let server = MockWorkerServer::consecutive_poll_failures(2);
22981        let client = Client::builder(server.base_url())
22982            .timeout(Duration::from_secs(2))
22983            .build()
22984            .expect("client");
22985        let mut worker = Worker::new(client, "capture")
22986            .worker_id("capture-worker")
22987            .poll_timeout(Duration::from_millis(10))
22988            .retry_policy(WorkerRetryPolicy {
22989                max_retries: 2,
22990                initial_backoff: Duration::from_millis(1),
22991                max_backoff: Duration::from_millis(1),
22992            });
22993        worker.register_workflow(
22994            "capture.workflow",
22995            |_ctx, _input| async move { Ok(Value::Null) },
22996        );
22997        worker.register_activity(
22998            "capture.activity",
22999            |_ctx, _input| async move { Ok(Value::Null) },
23000        );
23001        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23002            Ok(Value::Null)
23003        });
23004
23005        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23006
23007        for path in [
23008            "/api/worker/workflow-tasks/poll",
23009            "/api/worker/activity-tasks/poll",
23010            "/api/worker/query-tasks/poll",
23011        ] {
23012            let bodies = server.request_bodies(path);
23013            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23014            assert!(
23015                bodies
23016                    .iter()
23017                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23018                "{path} must preserve one request binding across every retry"
23019            );
23020        }
23021    }
23022
23023    fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23024        let reason = if unavailable {
23025            "storage_admission_unavailable"
23026        } else {
23027            "storage_pressure"
23028        };
23029        let mut body = json!({
23030            "reason": reason,
23031            "storage_state": if unavailable { "fenced" } else { "draining" },
23032            "retryable": true,
23033            "retry_after_seconds": 1,
23034        });
23035        if !mid_poll {
23036            body["request_admitted"] = json!(false);
23037        }
23038        if let Some(id) = poll_id {
23039            body["task"] = Value::Null;
23040            body["poll_status"] = json!(reason);
23041            body["poll_request_id"] = json!(id);
23042            body["retry_same_poll_request_id"] = json!(true);
23043            body["claim_admitted"] = json!(false);
23044        }
23045        body
23046    }
23047
23048    fn storage_worker(server: &MockWorkerServer) -> Worker {
23049        Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23050            .worker_id("storage-worker")
23051            .retry_policy(WorkerRetryPolicy {
23052                max_retries: 1,
23053                initial_backoff: Duration::from_millis(1),
23054                max_backoff: Duration::from_millis(1),
23055            })
23056    }
23057
23058    fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23059        let requests = server.requests.lock().expect("requests");
23060        let bodies: Vec<_> = requests
23061            .iter()
23062            .filter(|request| request.path == path)
23063            .map(|request| &request.body)
23064            .collect();
23065        assert_eq!(bodies.len(), count, "{path}");
23066        assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23067    }
23068
23069    #[test]
23070    fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23071        for unavailable in [false, true] {
23072            for mid_poll in [false, true] {
23073                let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23074                let error = Error::Http {
23075                    status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23076                    body: body.to_string(),
23077                };
23078                assert_eq!(
23079                    worker_storage_admission_retry_after(&error, Some("same-poll")),
23080                    Some(Duration::from_secs(1))
23081                );
23082                assert!(
23083                    !worker_operation_is_retryable(&error),
23084                    "storage is not a bounded generic retry"
23085                );
23086                for (field, value) in [
23087                    ("poll_request_id", json!("wrong-poll")),
23088                    ("task", json!({"task_id":"claimed"})),
23089                    ("retryable", json!(false)),
23090                    ("retry_after_seconds", json!(0)),
23091                    ("retry_after_seconds", json!(true)),
23092                    ("retry_after_seconds", json!(1.0)),
23093                    ("storage_state", json!("normal")),
23094                    ("poll_status", json!("empty")),
23095                    ("claim_admitted", json!(true)),
23096                    ("retry_same_poll_request_id", json!(false)),
23097                    ("request_admitted", json!(true)),
23098                ] {
23099                    let mut invalid = body.clone();
23100                    invalid[field] = value;
23101                    let error = Error::Http {
23102                        status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23103                        body: invalid.to_string(),
23104                    };
23105                    assert!(
23106                        worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23107                        "{field}"
23108                    );
23109                }
23110            }
23111        }
23112        let body = storage_refusal(None, false, false);
23113        let error = Error::Http {
23114            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23115            body: body.to_string(),
23116        };
23117        assert!(worker_storage_admission_retry_after(&error, None).is_some());
23118        assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23119        let error = Error::Http {
23120            status: reqwest::StatusCode::FORBIDDEN,
23121            body: body.to_string(),
23122        };
23123        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23124        let body = storage_refusal(None, false, true);
23125        let error = Error::Http {
23126            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23127            body: body.to_string(),
23128        };
23129        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23130    }
23131
23132    #[tokio::test]
23133    async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23134        for unavailable in [false, true] {
23135            for mid_poll in [false, true] {
23136                let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23137                    poll_failures_per_path: 1,
23138                    storage_refusals: 7,
23139                    storage_path: Some("/poll"),
23140                    storage_unavailable: unavailable,
23141                    storage_mid_poll: mid_poll,
23142                    ..MockWorkerBehavior::default()
23143                });
23144                let mut worker = storage_worker(&server);
23145                worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23146                assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23147                for path in [
23148                    "/api/worker/workflow-tasks/poll",
23149                    "/api/worker/activity-tasks/poll",
23150                    "/api/worker/query-tasks/poll",
23151                ] {
23152                    assert_identical_requests(&server, path, 9);
23153                }
23154            }
23155        }
23156    }
23157
23158    #[tokio::test]
23159    async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23160        struct CountedBody(Arc<AtomicUsize>);
23161        impl Serialize for CountedBody {
23162            fn serialize<S: Serializer>(
23163                &self,
23164                serializer: S,
23165            ) -> std::result::Result<S::Ok, S::Error> {
23166                let count = self.0.fetch_add(1, Ordering::SeqCst);
23167                json!({"serialization":count,"lease_owner":"worker","attempt":7})
23168                    .serialize(serializer)
23169            }
23170        }
23171        for path in [
23172            "/api/worker/register",
23173            "/api/worker/heartbeat",
23174            "/api/worker/workflow-tasks/storage-task/complete",
23175            "/api/worker/workflow-tasks/storage-task/fail",
23176            "/api/worker/activity-tasks/storage-task/complete",
23177            "/api/worker/activity-tasks/storage-task/fail",
23178            "/api/worker/activity-tasks/storage-task/heartbeat",
23179            "/api/worker/query-tasks/storage-task/complete",
23180            "/api/worker/query-tasks/storage-task/fail",
23181        ] {
23182            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23183                storage_refusals: 7,
23184                storage_path: Some(path),
23185                ..MockWorkerBehavior::default()
23186            });
23187            let worker =
23188                storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23189            let calls = Arc::new(AtomicUsize::new(0));
23190            let _: Value = worker
23191                .client
23192                .request_json(
23193                    reqwest::Method::POST,
23194                    &path[4..],
23195                    RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23196                    Some(&CountedBody(Arc::clone(&calls))),
23197                )
23198                .await
23199                .expect("prepared request recovery");
23200            assert_eq!(calls.load(Ordering::SeqCst), 1);
23201            assert_identical_requests(&server, path, 8);
23202        }
23203        for worker_scope in [false, true] {
23204            let path = if worker_scope {
23205                "/api/health"
23206            } else {
23207                "/api/worker/register"
23208            };
23209            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23210                storage_refusals: usize::MAX,
23211                storage_path: Some(path),
23212                ..MockWorkerBehavior::default()
23213            });
23214            let worker = storage_worker(&server);
23215            let client = worker.client.clone();
23216            let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
23217            let error = if worker_scope {
23218                worker
23219                    .client
23220                    .health()
23221                    .await
23222                    .expect_err("control plane is not retried")
23223            } else {
23224                client
23225                    .request_json::<Value, Value>(
23226                        reqwest::Method::POST,
23227                        "/worker/register",
23228                        RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23229                        Some(&json!({})),
23230                    )
23231                    .await
23232                    .expect_err("direct client is not retried")
23233            };
23234            assert!(worker_storage_admission_body(&error).is_some());
23235            assert_eq!(server.request_count(path), 1);
23236        }
23237    }
23238
23239    #[tokio::test]
23240    async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
23241        for fail in [false, true] {
23242            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23243                storage_activity: true,
23244                storage_refusals: 7,
23245                storage_path: Some("/storage-activity/"),
23246                ..MockWorkerBehavior::default()
23247            });
23248            let mut worker = storage_worker(&server);
23249            let calls = Arc::new(AtomicUsize::new(0));
23250            let observed = Arc::clone(&calls);
23251            worker.register_activity("storage.activity", move |ctx, _| {
23252                let calls = Arc::clone(&observed);
23253                async move {
23254                    calls.fetch_add(1, Ordering::SeqCst);
23255                    ctx.heartbeat(json!({"step":2})).await?;
23256                    if fail {
23257                        Err(Error::WorkerLoop("intentional handler failure".to_string()))
23258                    } else {
23259                        Ok(json!({"receipt":true}))
23260                    }
23261                }
23262            });
23263            assert_eq!(worker.run_once().await.expect("activity settled"), 1);
23264            assert_eq!(calls.load(Ordering::SeqCst), 1);
23265            assert_identical_requests(
23266                &server,
23267                "/api/worker/activity-tasks/storage-activity/heartbeat",
23268                8,
23269            );
23270            let suffix = if fail { "fail" } else { "complete" };
23271            assert_identical_requests(
23272                &server,
23273                &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
23274                8,
23275            );
23276            let other = if fail { "complete" } else { "fail" };
23277            assert_eq!(
23278                server.request_count(&format!(
23279                    "/api/worker/activity-tasks/storage-activity/{other}"
23280                )),
23281                0
23282            );
23283        }
23284    }
23285
23286    #[tokio::test]
23287    async fn storage_waits_are_interruptible_without_false_activity_failure() {
23288        for path in [
23289            "/api/worker/register",
23290            "/api/worker/heartbeat",
23291            "/api/worker/activity-tasks/poll",
23292            "/api/worker/activity-tasks/storage-activity/heartbeat",
23293            "/api/worker/activity-tasks/storage-activity/complete",
23294        ] {
23295            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23296                storage_activity: true,
23297                storage_refusals: usize::MAX,
23298                storage_path: Some(path),
23299                ..MockWorkerBehavior::default()
23300            });
23301            let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23302            worker.register_activity("storage.activity", |ctx, _| async move {
23303                ctx.heartbeat(json!({"step":2})).await?;
23304                Ok(json!({"receipt":true}))
23305            });
23306            let shutdown = async {
23307                while server.request_count(path) == 0 {
23308                    tokio::time::sleep(Duration::from_millis(1)).await;
23309                }
23310            };
23311            let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
23312                .await
23313                .expect("shutdown interrupts admission");
23314            assert!(
23315                result.is_err(),
23316                "a refused operation must not appear acknowledged: {path}, {result:?}"
23317            );
23318            assert_eq!(server.request_count(path), 1);
23319            assert_eq!(
23320                server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
23321                0
23322            );
23323            assert_eq!(
23324                server.request_count("/api/worker/registrations/mock-worker"),
23325                usize::from(!path.ends_with("/register"))
23326            );
23327        }
23328    }
23329
23330    #[tokio::test]
23331    async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
23332        for fail in [false, true] {
23333            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23334                storage_query: true,
23335                storage_refusals: 7,
23336                storage_path: Some("/storage-query/"),
23337                ..MockWorkerBehavior::default()
23338            });
23339            let mut worker = storage_worker(&server);
23340            worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
23341            let calls = Arc::new(AtomicUsize::new(0));
23342            let observed = Arc::clone(&calls);
23343            worker.register_query("storage.workflow", "state", move |_, _| {
23344                let calls = Arc::clone(&observed);
23345                async move {
23346                    calls.fetch_add(1, Ordering::SeqCst);
23347                    if fail {
23348                        Err(Error::WorkerLoop("intentional query failure".to_string()))
23349                    } else {
23350                        Ok(json!({"state":"waiting"}))
23351                    }
23352                }
23353            });
23354            assert_eq!(worker.run_once().await.expect("query settled"), 1);
23355            assert_eq!(calls.load(Ordering::SeqCst), 1);
23356            let suffix = if fail { "fail" } else { "complete" };
23357            assert_identical_requests(
23358                &server,
23359                &format!("/api/worker/query-tasks/storage-query/{suffix}"),
23360                8,
23361            );
23362            let other = if fail { "complete" } else { "fail" };
23363            assert_eq!(
23364                server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
23365                0
23366            );
23367        }
23368    }
23369
23370    #[tokio::test]
23371    async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
23372        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23373            storage_refusals: 7,
23374            storage_path: Some("/poll"),
23375            unauthorized_polls: true,
23376            ..MockWorkerBehavior::default()
23377        });
23378        let error = storage_worker(&server)
23379            .run_once()
23380            .await
23381            .expect_err("auth remains terminal");
23382        assert!(matches!(
23383            error,
23384            Error::Http {
23385                status: reqwest::StatusCode::UNAUTHORIZED,
23386                ..
23387            }
23388        ));
23389        assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
23390
23391        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23392            storage_refusals: 7,
23393            storage_path: Some("/activity-cancel/complete"),
23394            ..MockWorkerBehavior::default()
23395        });
23396        let worker =
23397            storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23398        let error = worker
23399            .client
23400            .complete_activity_task(
23401                "activity-cancel",
23402                "attempt-cancel",
23403                "worker",
23404                json!({}),
23405                DEFAULT_CODEC,
23406            )
23407            .await
23408            .expect_err("cancellation remains terminal");
23409        assert!(activity_task_rejection_is_final(&error));
23410        assert_identical_requests(
23411            &server,
23412            "/api/worker/activity-tasks/activity-cancel/complete",
23413            8,
23414        );
23415
23416        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23417            storage_refusals: usize::MAX,
23418            storage_path: Some("/poll"),
23419            storage_wrong_poll_id: true,
23420            ..MockWorkerBehavior::default()
23421        });
23422        assert!(storage_worker(&server).run_once().await.is_err());
23423        assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
23424    }
23425
23426    #[tokio::test]
23427    async fn storage_pollers_stop_when_the_run_future_is_aborted() {
23428        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23429            storage_refusals: usize::MAX,
23430            storage_path: Some("/poll"),
23431            ..MockWorkerBehavior::default()
23432        });
23433        let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23434        worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
23435        let run = tokio::spawn(async move { worker.run().await });
23436        tokio::time::timeout(Duration::from_secs(2), async {
23437            while server.request_count("/api/worker/activity-tasks/poll") == 0 {
23438                tokio::time::sleep(Duration::from_millis(1)).await;
23439            }
23440        })
23441        .await
23442        .expect("poll started");
23443        run.abort();
23444        assert!(run.await.expect_err("cancelled run").is_cancelled());
23445        tokio::time::sleep(Duration::from_millis(250)).await;
23446        assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
23447    }
23448
23449    #[tokio::test]
23450    async fn query_protocol_rejection_from_older_server_is_typed() {
23451        let server = MockWorkerServer::reject_query_protocol();
23452        let client = Client::builder(server.base_url())
23453            .timeout(Duration::from_secs(2))
23454            .build()
23455            .expect("client");
23456
23457        let error = client
23458            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23459            .await
23460            .expect_err("server below query protocol floor must reject");
23461        let Error::Protocol(failure) = error else {
23462            panic!("expected typed protocol failure");
23463        };
23464
23465        assert_eq!(failure.status, 400);
23466        assert_eq!(failure.reason, "unsupported_protocol_version");
23467        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
23468        assert_eq!(
23469            failure.requested_version.as_deref(),
23470            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23471        );
23472        assert_eq!(
23473            server
23474                .worker_protocol_for("/api/worker/query-tasks/poll")
23475                .as_deref(),
23476            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23477        );
23478    }
23479
23480    #[tokio::test]
23481    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
23482        let server = MockWorkerServer::reject_query_protocol();
23483        let client = Client::builder(server.base_url())
23484            .timeout(Duration::from_secs(2))
23485            .build()
23486            .expect("client");
23487        let mut worker = Worker::new(client, "rust-workers")
23488            .worker_id("baseline-worker")
23489            .poll_timeout(Duration::from_millis(10));
23490
23491        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
23492            Ok(Value::Null)
23493        });
23494
23495        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
23496        assert_eq!(
23497            server
23498                .worker_protocol_for("/api/worker/workflow-tasks/poll")
23499                .as_deref(),
23500            Some(WORKER_PROTOCOL_VERSION)
23501        );
23502        assert_eq!(
23503            server.worker_protocol_for("/api/worker/query-tasks/poll"),
23504            None,
23505            "a worker without query handlers must not use the query-task endpoint"
23506        );
23507    }
23508
23509    #[tokio::test]
23510    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
23511        let server = MockWorkerServer::reject_query_completion();
23512        let client = Client::builder(server.base_url())
23513            .timeout(Duration::from_secs(2))
23514            .build()
23515            .expect("client");
23516
23517        let error = client
23518            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
23519            .await
23520            .expect_err("expired completion must be rejected");
23521        let Error::QueryFailed(failure) = error else {
23522            panic!("expected typed query failure");
23523        };
23524        assert_eq!(failure.status, 409);
23525        assert_eq!(failure.reason, "query_task_timed_out");
23526
23527        let mut worker = Worker::new(client, "rust-workers")
23528            .worker_id("late-worker")
23529            .poll_timeout(Duration::from_millis(10));
23530        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23531        worker.register_query(
23532            "counter",
23533            "current",
23534            |_ctx, _args| async move { Ok(json!(8)) },
23535        );
23536
23537        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
23538        assert_eq!(
23539            worker
23540                .run_once()
23541                .await
23542                .expect("worker continues after late completion"),
23543            0
23544        );
23545        assert_eq!(
23546            server.request_count("/api/worker/query-tasks/query-late/complete"),
23547            2
23548        );
23549        assert_eq!(
23550            server.request_count("/api/worker/query-tasks/query-late/fail"),
23551            0,
23552            "a server completion rejection must not be reported as an encoding failure"
23553        );
23554    }
23555
23556    #[tokio::test]
23557    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
23558        let server = MockWorkerServer::start();
23559        let client = Client::builder(server.base_url())
23560            .timeout(Duration::from_secs(2))
23561            .build()
23562            .expect("client");
23563        let mut worker = Worker::new(client, "rust-workers")
23564            .worker_id("joined-worker")
23565            .poll_timeout(Duration::from_millis(10));
23566        worker.register_workflow(
23567            "joined.workflow",
23568            |_ctx, _input| async move { Ok(Value::Null) },
23569        );
23570        worker.register_activity(
23571            "joined.activity",
23572            |_ctx, _input| async move { Ok(Value::Null) },
23573        );
23574        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
23575            Ok(Value::Null)
23576        });
23577
23578        worker
23579            .run_until(tokio::time::sleep(Duration::from_millis(20)))
23580            .await
23581            .expect("normal shutdown");
23582
23583        let deregistration_path = "/api/worker/registrations/mock-worker";
23584        assert_eq!(server.request_count(deregistration_path), 1);
23585        for poll_path in [
23586            "/api/worker/workflow-tasks/poll",
23587            "/api/worker/activity-tasks/poll",
23588            "/api/worker/query-tasks/poll",
23589        ] {
23590            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
23591        }
23592        assert_eq!(
23593            server.captured_paths().last().map(String::as_str),
23594            Some(deregistration_path),
23595            "deregistration must start only after every poller has joined"
23596        );
23597    }
23598
23599    #[tokio::test]
23600    async fn registration_failure_does_not_deregister() {
23601        let server = MockWorkerServer::rejected_registration();
23602        let client = Client::builder(server.base_url())
23603            .timeout(Duration::from_secs(2))
23604            .build()
23605            .expect("client");
23606        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
23607
23608        let error = worker
23609            .run_until(async {})
23610            .await
23611            .expect_err("registration must fail");
23612        assert!(matches!(
23613            error,
23614            Error::Http {
23615                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23616                ..
23617            }
23618        ));
23619        assert!(server
23620            .captured_paths()
23621            .iter()
23622            .all(|path| !path.starts_with("/api/worker/registrations/")));
23623    }
23624
23625    #[tokio::test]
23626    async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
23627        let server = MockWorkerServer::rejected_registration_protocol();
23628        let client = Client::builder(server.base_url())
23629            .timeout(Duration::from_secs(2))
23630            .build()
23631            .expect("client");
23632        let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
23633
23634        let error = worker
23635            .run_until(async {})
23636            .await
23637            .expect_err("a protocol 1.16 server must reject this worker");
23638        let Error::Protocol(failure) = error else {
23639            panic!("expected typed protocol rejection");
23640        };
23641        assert_eq!(failure.reason, "unsupported_protocol_version");
23642        assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
23643        assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
23644        assert_eq!(
23645            server
23646                .worker_protocol_for("/api/worker/register")
23647                .as_deref(),
23648            Some(WORKER_PROTOCOL_VERSION)
23649        );
23650    }
23651
23652    #[tokio::test]
23653    async fn declined_registration_does_not_deregister() {
23654        let server = MockWorkerServer::declined_registration();
23655        let client = Client::builder(server.base_url())
23656            .timeout(Duration::from_secs(2))
23657            .build()
23658            .expect("client");
23659        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
23660
23661        let error = worker
23662            .run_until(async {})
23663            .await
23664            .expect_err("declined registration must fail");
23665        assert!(matches!(error, Error::WorkerLoop(_)));
23666        assert!(error.to_string().contains("was not accepted"));
23667        assert!(server
23668            .captured_paths()
23669            .iter()
23670            .all(|path| !path.starts_with("/api/worker/registrations/")));
23671    }
23672
23673    #[tokio::test]
23674    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23675        let server = MockWorkerServer::rejected_deregistration();
23676        let client = Client::builder(server.base_url())
23677            .timeout(Duration::from_secs(2))
23678            .build()
23679            .expect("client");
23680        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23681
23682        let error = worker
23683            .run_until(async {})
23684            .await
23685            .expect_err("deregistration must fail");
23686        assert!(matches!(
23687            error,
23688            Error::Http {
23689                status: reqwest::StatusCode::FORBIDDEN,
23690                ..
23691            }
23692        ));
23693        assert_eq!(
23694            server.request_count("/api/worker/registrations/mock-worker"),
23695            1
23696        );
23697    }
23698
23699    #[tokio::test]
23700    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23701        let server = MockWorkerServer::rejected_deregistration_protocol();
23702        let client = Client::builder(server.base_url())
23703            .timeout(Duration::from_secs(2))
23704            .build()
23705            .expect("client");
23706        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
23707
23708        let error = worker
23709            .run_until(async {})
23710            .await
23711            .expect_err("protocol rejection must fail shutdown");
23712        let Error::Protocol(failure) = error else {
23713            panic!("expected typed protocol failure");
23714        };
23715        assert_eq!(failure.reason, "unsupported_protocol_version");
23716        assert_eq!(
23717            failure.requested_version.as_deref(),
23718            Some(WORKER_PROTOCOL_VERSION)
23719        );
23720        assert_eq!(
23721            server.request_count("/api/worker/registrations/mock-worker"),
23722            1
23723        );
23724    }
23725
23726    #[tokio::test]
23727    async fn primary_poller_error_retains_deregistration_failure_context() {
23728        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
23729        let client = Client::builder(server.base_url())
23730            .timeout(Duration::from_secs(2))
23731            .build()
23732            .expect("client");
23733        let mut worker = Worker::new(client, "rust-workers")
23734            .worker_id("combined-failure")
23735            .poll_timeout(Duration::from_millis(10));
23736        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
23737            Ok(Value::Null)
23738        });
23739
23740        let error = worker
23741            .run()
23742            .await
23743            .expect_err("worker and cleanup must fail");
23744        let summary = error.to_string();
23745        assert!(summary.contains("authentication_failed"));
23746        assert!(summary.contains("worker cannot deregister"));
23747        let Error::WorkerShutdown {
23748            primary,
23749            deregistration,
23750        } = error
23751        else {
23752            panic!("expected combined worker shutdown error");
23753        };
23754        assert!(matches!(
23755            *primary,
23756            Error::Http {
23757                status: reqwest::StatusCode::UNAUTHORIZED,
23758                ..
23759            }
23760        ));
23761        assert!(matches!(
23762            *deregistration,
23763            Error::Http {
23764                status: reqwest::StatusCode::FORBIDDEN,
23765                ..
23766            }
23767        ));
23768        assert_eq!(
23769            server.request_count("/api/worker/registrations/mock-worker"),
23770            1
23771        );
23772    }
23773
23774    #[tokio::test]
23775    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
23776        let server = MockWorkerServer::start();
23777        let client = Client::builder(server.base_url())
23778            .timeout(Duration::from_secs(2))
23779            .build()
23780            .expect("client");
23781        let mut worker = Worker::new(client, "rust-workers")
23782            .worker_id("activity-only-worker")
23783            .poll_timeout(Duration::from_millis(10));
23784
23785        worker.register_activity(
23786            "activity.only",
23787            |_ctx, _args| async move { Ok(Value::Null) },
23788        );
23789
23790        worker.run_until(async {}).await.expect("run worker");
23791    }
23792
23793    #[tokio::test]
23794    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
23795        let server = MockWorkerServer::start();
23796        let client = Client::builder(server.base_url())
23797            .timeout(Duration::from_secs(2))
23798            .build()
23799            .expect("client");
23800        let mut worker = Worker::new(client, "rust-workers")
23801            .worker_id("workflow-only-worker")
23802            .poll_timeout(Duration::from_millis(10));
23803
23804        worker.register_workflow(
23805            "workflow.only",
23806            |_ctx, _input| async move { Ok(Value::Null) },
23807        );
23808
23809        worker.run_until(async {}).await.expect("run worker");
23810    }
23811
23812    #[tokio::test]
23813    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
23814        let server = MockWorkerServer::start();
23815        let client = Client::builder(server.base_url())
23816            .timeout(Duration::from_secs(2))
23817            .build()
23818            .expect("client");
23819        let observations = Arc::new(Mutex::new(Vec::new()));
23820        let observed = Arc::clone(&observations);
23821        let mut worker = Worker::new(client, "rust-workers")
23822            .worker_id("observed-heartbeat-worker")
23823            .poll_timeout(Duration::from_millis(10))
23824            .on_worker_heartbeat(move |observation| {
23825                observed
23826                    .lock()
23827                    .expect("heartbeat observations")
23828                    .push(observation.clone());
23829            });
23830
23831        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
23832            Ok(Value::Null)
23833        });
23834        let acknowledged = Arc::clone(&observations);
23835        worker
23836            .run_until(async move {
23837                tokio::time::timeout(Duration::from_secs(2), async move {
23838                    loop {
23839                        if !acknowledged
23840                            .lock()
23841                            .expect("heartbeat observations")
23842                            .is_empty()
23843                        {
23844                            break;
23845                        }
23846                        tokio::time::sleep(Duration::from_millis(1)).await;
23847                    }
23848                })
23849                .await
23850                .expect("heartbeat acknowledgement within timeout");
23851            })
23852            .await
23853            .expect("run worker");
23854
23855        let observations = observations.lock().expect("heartbeat observations");
23856        let first = observations.first().expect("heartbeat acknowledgement");
23857        assert_eq!(first.worker_id, "observed-heartbeat-worker");
23858        assert_eq!(first.task_queue, "rust-workers");
23859        assert!(first.acknowledged_at_unix_millis > 0);
23860        assert_eq!(first.acknowledgement, json!({}));
23861    }
23862
23863    #[tokio::test]
23864    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
23865        let server = MockWorkerServer::delayed_heartbeat_worker();
23866        let client = Client::builder(server.base_url())
23867            .timeout(Duration::from_secs(3))
23868            .build()
23869            .expect("client");
23870        let observations = Arc::new(Mutex::new(Vec::new()));
23871        let observed = Arc::clone(&observations);
23872        let mut worker = Worker::new(client, "rust-snapshot-workers")
23873            .worker_id("rust-snapshot-worker")
23874            .poll_timeout(Duration::from_millis(10))
23875            .on_worker_heartbeat(move |observation| {
23876                observed
23877                    .lock()
23878                    .expect("heartbeat observations")
23879                    .push(observation.clone());
23880            });
23881
23882        worker.register_workflow("snapshot", |ctx, _input| async move {
23883            ctx.wait_signal("finish").await?;
23884            Ok(json!({"status": "finished"}))
23885        });
23886        worker.register_query("snapshot", "current", |ctx, _args| async move {
23887            Ok(json!(ctx
23888                .signals("increment")
23889                .iter()
23890                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23891                .sum::<i64>()))
23892        });
23893        worker.register_activity("cancel-aware", |_ctx, _args| async move {
23894            Ok(json!({"late": "completion"}))
23895        });
23896
23897        worker
23898            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
23899            .await
23900            .expect("delayed heartbeat must allow a clean worker shutdown");
23901
23902        let observations = observations.lock().expect("heartbeat observations");
23903        assert!(
23904            observations.len() >= 3,
23905            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
23906        );
23907        assert!(
23908            observations.windows(2).all(|pair| {
23909                pair[1].acknowledged_at_unix_millis
23910                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
23911                    >= 850
23912            }),
23913            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
23914        );
23915        drop(observations);
23916
23917        let heartbeat_times = server.request_times("/api/worker/heartbeat");
23918        let delayed_request_at = *heartbeat_times
23919            .get(1)
23920            .expect("intentionally delayed heartbeat request");
23921        let delay_window_start = delayed_request_at + Duration::from_millis(100);
23922        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
23923        for path in [
23924            "/api/worker/workflow-tasks/poll",
23925            "/api/worker/activity-tasks/poll",
23926            "/api/worker/query-tasks/poll",
23927        ] {
23928            assert!(
23929                server
23930                    .request_times(path)
23931                    .iter()
23932                    .any(|received_at| *received_at >= delay_window_start
23933                        && *received_at <= delay_window_end),
23934                "{path} must keep polling while a heartbeat acknowledgement is delayed"
23935            );
23936        }
23937        assert!(
23938            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
23939            "workflow work must be settled"
23940        );
23941        assert!(
23942            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
23943            "activity work must be settled"
23944        );
23945        assert!(
23946            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
23947            "query work must be settled"
23948        );
23949    }
23950
23951    #[tokio::test]
23952    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
23953        let server = MockWorkerServer::heartbeat_retry_worker();
23954        let client = Client::builder(server.base_url())
23955            .timeout(Duration::from_secs(2))
23956            .build()
23957            .expect("client");
23958        let observations = Arc::new(Mutex::new(Vec::new()));
23959        let observed = Arc::clone(&observations);
23960        let worker = Worker::new(client, "rust-workers")
23961            .worker_id("heartbeat-retry-worker")
23962            .retry_policy(WorkerRetryPolicy {
23963                max_retries: 1,
23964                initial_backoff: Duration::from_millis(300),
23965                max_backoff: Duration::from_millis(300),
23966            })
23967            .on_worker_heartbeat(move |observation| {
23968                observed
23969                    .lock()
23970                    .expect("heartbeat observations")
23971                    .push(observation.clone());
23972            });
23973
23974        worker
23975            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
23976            .await
23977            .expect("retryable heartbeat failure must remain bounded and recover");
23978
23979        let observations = observations.lock().expect("heartbeat observations");
23980        assert!(observations.len() >= 3, "heartbeat retry must recover");
23981        assert!(
23982            observations.windows(2).all(|pair| {
23983                pair[1]
23984                    .acknowledged_at_unix_millis
23985                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
23986                    >= 850
23987            }),
23988            "a successful retry must start a fresh advertised cadence: {observations:?}"
23989        );
23990        assert_eq!(
23991            server.request_count("/api/worker/heartbeat"),
23992            observations.len() + 1,
23993            "one retryable failure must add exactly one bounded request"
23994        );
23995    }
23996
23997    #[tokio::test]
23998    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
23999        let server = MockWorkerServer::waiting_query_worker();
24000        let client = Client::builder(server.base_url())
24001            .timeout(Duration::from_secs(2))
24002            .build()
24003            .expect("client");
24004        let observations = Arc::new(Mutex::new(Vec::new()));
24005        let observed = Arc::clone(&observations);
24006        let mut worker = Worker::new(client, "rust-snapshot-workers")
24007            .worker_id("rust-snapshot-worker")
24008            .poll_timeout(Duration::from_millis(10))
24009            .on_worker_heartbeat(move |observation| {
24010                observed
24011                    .lock()
24012                    .expect("heartbeat observations")
24013                    .push(observation.clone());
24014            });
24015
24016        worker.register_workflow("snapshot", |ctx, _input| async move {
24017            ctx.wait_signal("finish").await?;
24018            Ok(json!({"status": "finished"}))
24019        });
24020        worker.register_query("snapshot", "current", |ctx, _args| async move {
24021            let current = ctx
24022                .signals("increment")
24023                .iter()
24024                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24025                .sum::<i64>();
24026            Ok(json!(current))
24027        });
24028        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24029
24030        worker
24031            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24032            .await
24033            .expect("pending workflow and query poller must remain live until shutdown");
24034
24035        assert!(
24036            observations.lock().expect("heartbeat observations").len() >= 4,
24037            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24038        );
24039        assert!(
24040            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24041            "workflow polling must continue after empty replay acknowledgements"
24042        );
24043        assert!(
24044            server.request_count("/api/worker/query-tasks/poll") >= 2,
24045            "query polling must continue after serving the current query"
24046        );
24047        assert_eq!(
24048            server.request_body("/api/worker/register")["capabilities"],
24049            json!([
24050                CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24051                DURABLE_SELECTION_CAPABILITY,
24052                MEMO_UPSERTS_CAPABILITY,
24053                TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24054                QUERY_TASKS_CAPABILITY,
24055                WORKFLOW_UPDATES_CAPABILITY,
24056                MESSAGE_STREAMS_CAPABILITY
24057            ])
24058        );
24059        assert_eq!(
24060            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24061            json!({
24062                "queries": ["current"],
24063                "query_contracts": [],
24064                "signals": [],
24065                "signal_contracts": [],
24066                "updates": ["replace"],
24067                "update_contracts": [],
24068                "update_validators": [],
24069            })
24070        );
24071
24072        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24073        assert_eq!(
24074            opened["commands"],
24075            json!([{
24076                "type": "open_signal_wait",
24077                "signal_name": "finish",
24078            }])
24079        );
24080
24081        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24082            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24083            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24084            let failure = server.request_body(&fail_path);
24085            assert_eq!(
24086                failure["failure"]["type"],
24087                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24088            );
24089            assert_eq!(server.request_count(&completion_path), 0);
24090        }
24091
24092        let query_completion =
24093            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24094        assert_eq!(query_completion["result"], json!(8));
24095
24096        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24097        assert_eq!(
24098            server.request_count(terminal_path),
24099            1,
24100            "the matching signal must settle the workflow exactly once"
24101        );
24102        let terminal = server.request_body(terminal_path);
24103        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24104        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24105        assert_eq!(
24106            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24107                .expect("terminal workflow result"),
24108            json!({"status": "finished"})
24109        );
24110    }
24111
24112    #[tokio::test]
24113    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24114        let server = MockWorkerServer::transient_worker_failures();
24115        let client = Client::builder(server.base_url())
24116            .timeout(Duration::from_secs(2))
24117            .build()
24118            .expect("client");
24119        let mut worker = Worker::new(client, "rust-workers")
24120            .worker_id("retry-worker")
24121            .poll_timeout(Duration::from_millis(10))
24122            .retry_policy(WorkerRetryPolicy {
24123                max_retries: 2,
24124                initial_backoff: Duration::from_millis(1),
24125                max_backoff: Duration::from_millis(1),
24126            });
24127        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24128        worker.register_activity(
24129            "counter.activity",
24130            |_ctx, _input| async move { Ok(Value::Null) },
24131        );
24132        worker.register_query(
24133            "counter",
24134            "current",
24135            |_ctx, _args| async move { Ok(json!(8)) },
24136        );
24137
24138        worker
24139            .run_until(tokio::time::sleep(Duration::from_millis(75)))
24140            .await
24141            .expect("transient failures must not stop the worker");
24142
24143        for path in [
24144            "/api/worker/heartbeat",
24145            "/api/worker/workflow-tasks/poll",
24146            "/api/worker/activity-tasks/poll",
24147            "/api/worker/query-tasks/poll",
24148        ] {
24149            assert!(
24150                server.request_count(path) >= 2,
24151                "{path} must continue after its transient failure"
24152            );
24153        }
24154    }
24155
24156    #[tokio::test]
24157    async fn worker_continues_after_long_poll_capacity_backpressure() {
24158        let server = MockWorkerServer::capacity_limited_activity_poll();
24159        let client = Client::builder(server.base_url())
24160            .timeout(Duration::from_secs(2))
24161            .build()
24162            .expect("client");
24163        let mut worker = Worker::new(client, "rust-workers")
24164            .worker_id("capacity-worker")
24165            .poll_timeout(Duration::from_millis(10))
24166            .retry_policy(WorkerRetryPolicy {
24167                max_retries: 0,
24168                initial_backoff: Duration::from_millis(1),
24169                max_backoff: Duration::from_millis(1),
24170            });
24171        worker.register_activity("capacity.activity", |_ctx, _input| async move {
24172            Ok(json!({"handled": true}))
24173        });
24174
24175        worker
24176            .run_until(tokio::time::sleep(Duration::from_millis(50)))
24177            .await
24178            .expect("capacity backpressure must not stop the worker");
24179
24180        assert!(
24181            server.request_count("/api/worker/activity-tasks/poll") >= 2,
24182            "the activity poller must continue after capacity backpressure"
24183        );
24184        assert_eq!(
24185            server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24186            1,
24187            "the worker must complete work returned after capacity recovers"
24188        );
24189    }
24190
24191    #[test]
24192    fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
24193        let capacity = Error::Http {
24194            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24195            body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
24196        };
24197        assert_eq!(
24198            worker_poll_capacity_retry_after(&capacity),
24199            Some(Duration::from_secs(3))
24200        );
24201
24202        let rejected_capacity = Error::Http {
24203            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24204            body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
24205        };
24206        assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
24207        assert!(!worker_operation_is_retryable(&rejected_capacity));
24208
24209        let ordinary_rate_limit = Error::Http {
24210            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24211            body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
24212                .to_string(),
24213        };
24214        assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
24215        assert!(worker_operation_is_retryable(&ordinary_rate_limit));
24216    }
24217
24218    #[tokio::test]
24219    async fn worker_bounds_transport_retries() {
24220        let server = MockWorkerServer::unavailable_polls();
24221        let client = Client::builder(server.base_url())
24222            .timeout(Duration::from_secs(2))
24223            .build()
24224            .expect("client");
24225        let mut worker = Worker::new(client, "rust-workers")
24226            .worker_id("bounded-retry-worker")
24227            .poll_timeout(Duration::from_millis(10))
24228            .retry_policy(WorkerRetryPolicy {
24229                max_retries: 2,
24230                initial_backoff: Duration::from_millis(1),
24231                max_backoff: Duration::from_millis(1),
24232            });
24233        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24234
24235        let error = worker.run().await.expect_err("retry bound must terminate");
24236        assert!(matches!(error, Error::Transport(_)));
24237        assert_eq!(
24238            server.request_count("/api/worker/workflow-tasks/poll"),
24239            3,
24240            "one initial request plus exactly two retries"
24241        );
24242    }
24243
24244    #[tokio::test]
24245    async fn worker_retry_policy_can_disable_poll_retries() {
24246        let server = MockWorkerServer::unavailable_polls();
24247        let client = Client::builder(server.base_url())
24248            .timeout(Duration::from_secs(2))
24249            .build()
24250            .expect("client");
24251        let mut worker = Worker::new(client, "rust-workers")
24252            .worker_id("no-retry-worker")
24253            .poll_timeout(Duration::from_millis(10))
24254            .retry_policy(WorkerRetryPolicy {
24255                max_retries: 0,
24256                initial_backoff: Duration::from_millis(1),
24257                max_backoff: Duration::from_millis(1),
24258            });
24259        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24260
24261        let error = worker
24262            .run_once()
24263            .await
24264            .expect_err("disabled retries must return the first transport failure");
24265        assert!(matches!(error, Error::Transport(_)));
24266        assert_eq!(
24267            server.request_count("/api/worker/workflow-tasks/poll"),
24268            1,
24269            "max_retries=0 must send only the initial request"
24270        );
24271    }
24272
24273    #[tokio::test]
24274    async fn worker_does_not_retry_authentication_failures() {
24275        let server = MockWorkerServer::unauthorized_polls();
24276        let client = Client::builder(server.base_url())
24277            .timeout(Duration::from_secs(2))
24278            .build()
24279            .expect("client");
24280        let mut worker = Worker::new(client, "rust-workers")
24281            .worker_id("unauthorized-worker")
24282            .poll_timeout(Duration::from_millis(10));
24283        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24284
24285        let error = worker
24286            .run()
24287            .await
24288            .expect_err("authentication must terminate");
24289        let Error::Http { status, body } = error else {
24290            panic!("expected stable HTTP authentication error");
24291        };
24292        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
24293        assert!(body.contains("authentication_failed"));
24294        assert_eq!(
24295            server.request_count("/api/worker/workflow-tasks/poll"),
24296            1,
24297            "authentication failures must not be retried"
24298        );
24299    }
24300
24301    #[derive(Clone, Debug)]
24302    struct CapturedRequest {
24303        method: String,
24304        path: String,
24305        authorization: Option<String>,
24306        namespace: Option<String>,
24307        worker_protocol: Option<String>,
24308        control_protocol: Option<String>,
24309        body: String,
24310        received_at: Instant,
24311    }
24312
24313    struct MockWorkerServer {
24314        addr: SocketAddr,
24315        stop: Arc<AtomicBool>,
24316        requests: Arc<Mutex<Vec<CapturedRequest>>>,
24317        thread: Option<thread::JoinHandle<()>>,
24318    }
24319
24320    #[derive(Clone, Copy, Default)]
24321    struct MockWorkerBehavior {
24322        storage_refusals: usize,
24323        storage_path: Option<&'static str>,
24324        storage_unavailable: bool,
24325        storage_mid_poll: bool,
24326        storage_activity: bool,
24327        storage_query: bool,
24328        storage_wrong_poll_id: bool,
24329        reject_query_protocol: bool,
24330        reject_query_completion: bool,
24331        waiting_query_worker: bool,
24332        decline_registration: bool,
24333        complete_named_signal: bool,
24334        poll_failures_per_path: usize,
24335        long_poll_capacity_responses_per_path: usize,
24336        heartbeat_failures: usize,
24337        heartbeat_failure_request: Option<usize>,
24338        delayed_heartbeat_request: Option<usize>,
24339        heartbeat_response_delay: Duration,
24340        concurrent_requests: bool,
24341        unauthorized_polls: bool,
24342        reject_registration: bool,
24343        reject_registration_protocol: bool,
24344        reject_deregistration: bool,
24345        reject_deregistration_protocol: bool,
24346        cancelled_activity: bool,
24347        draining_polls: bool,
24348        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
24349        workflow_completion_status: Option<&'static str>,
24350        workflow_completion_body: Option<&'static str>,
24351    }
24352
24353    impl MockWorkerServer {
24354        fn start() -> Self {
24355            Self::start_with_behavior(MockWorkerBehavior::default())
24356        }
24357
24358        fn reject_query_protocol() -> Self {
24359            Self::start_with_behavior(MockWorkerBehavior {
24360                reject_query_protocol: true,
24361                ..MockWorkerBehavior::default()
24362            })
24363        }
24364
24365        fn reject_query_completion() -> Self {
24366            Self::start_with_behavior(MockWorkerBehavior {
24367                reject_query_completion: true,
24368                ..MockWorkerBehavior::default()
24369            })
24370        }
24371
24372        fn waiting_query_worker() -> Self {
24373            Self::start_with_behavior(MockWorkerBehavior {
24374                waiting_query_worker: true,
24375                complete_named_signal: true,
24376                ..MockWorkerBehavior::default()
24377            })
24378        }
24379
24380        fn transient_worker_failures() -> Self {
24381            Self::start_with_behavior(MockWorkerBehavior {
24382                poll_failures_per_path: 1,
24383                heartbeat_failures: 1,
24384                ..MockWorkerBehavior::default()
24385            })
24386        }
24387
24388        fn consecutive_poll_failures(count: usize) -> Self {
24389            Self::start_with_behavior(MockWorkerBehavior {
24390                poll_failures_per_path: count,
24391                ..MockWorkerBehavior::default()
24392            })
24393        }
24394
24395        fn capacity_limited_activity_poll() -> Self {
24396            Self::start_with_behavior(MockWorkerBehavior {
24397                long_poll_capacity_responses_per_path: 1,
24398                ..MockWorkerBehavior::default()
24399            })
24400        }
24401
24402        fn delayed_heartbeat_worker() -> Self {
24403            Self::start_with_behavior(MockWorkerBehavior {
24404                waiting_query_worker: true,
24405                delayed_heartbeat_request: Some(2),
24406                heartbeat_response_delay: Duration::from_millis(1_500),
24407                concurrent_requests: true,
24408                cancelled_activity: true,
24409                ..MockWorkerBehavior::default()
24410            })
24411        }
24412
24413        fn heartbeat_retry_worker() -> Self {
24414            Self::start_with_behavior(MockWorkerBehavior {
24415                waiting_query_worker: true,
24416                heartbeat_failure_request: Some(2),
24417                concurrent_requests: true,
24418                ..MockWorkerBehavior::default()
24419            })
24420        }
24421
24422        fn unavailable_polls() -> Self {
24423            Self::start_with_behavior(MockWorkerBehavior {
24424                poll_failures_per_path: usize::MAX,
24425                ..MockWorkerBehavior::default()
24426            })
24427        }
24428
24429        fn unauthorized_polls() -> Self {
24430            Self::start_with_behavior(MockWorkerBehavior {
24431                unauthorized_polls: true,
24432                ..MockWorkerBehavior::default()
24433            })
24434        }
24435
24436        fn rejected_registration() -> Self {
24437            Self::start_with_behavior(MockWorkerBehavior {
24438                reject_registration: true,
24439                ..MockWorkerBehavior::default()
24440            })
24441        }
24442
24443        fn rejected_registration_protocol() -> Self {
24444            Self::start_with_behavior(MockWorkerBehavior {
24445                reject_registration_protocol: true,
24446                ..MockWorkerBehavior::default()
24447            })
24448        }
24449
24450        fn declined_registration() -> Self {
24451            Self::start_with_behavior(MockWorkerBehavior {
24452                decline_registration: true,
24453                ..MockWorkerBehavior::default()
24454            })
24455        }
24456
24457        fn rejected_deregistration() -> Self {
24458            Self::start_with_behavior(MockWorkerBehavior {
24459                reject_deregistration: true,
24460                ..MockWorkerBehavior::default()
24461            })
24462        }
24463
24464        fn rejected_deregistration_protocol() -> Self {
24465            Self::start_with_behavior(MockWorkerBehavior {
24466                reject_deregistration_protocol: true,
24467                ..MockWorkerBehavior::default()
24468            })
24469        }
24470
24471        fn unauthorized_polls_and_rejected_deregistration() -> Self {
24472            Self::start_with_behavior(MockWorkerBehavior {
24473                unauthorized_polls: true,
24474                reject_deregistration: true,
24475                ..MockWorkerBehavior::default()
24476            })
24477        }
24478
24479        fn cancelled_activity() -> Self {
24480            Self::start_with_behavior(MockWorkerBehavior {
24481                cancelled_activity: true,
24482                ..MockWorkerBehavior::default()
24483            })
24484        }
24485
24486        fn draining_polls() -> Self {
24487            Self::start_with_behavior(MockWorkerBehavior {
24488                draining_polls: true,
24489                ..MockWorkerBehavior::default()
24490            })
24491        }
24492
24493        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
24494            Self::start_with_behavior(MockWorkerBehavior {
24495                invalid_task_payload_codec: Some(codec),
24496                ..MockWorkerBehavior::default()
24497            })
24498        }
24499
24500        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
24501            Self::start_with_behavior(MockWorkerBehavior {
24502                workflow_completion_status: Some(status),
24503                workflow_completion_body: Some(body),
24504                ..MockWorkerBehavior::default()
24505            })
24506        }
24507
24508        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
24509            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
24510            listener
24511                .set_nonblocking(true)
24512                .expect("configure mock listener");
24513            let addr = listener.local_addr().expect("mock server address");
24514            let stop = Arc::new(AtomicBool::new(false));
24515            let server_stop = Arc::clone(&stop);
24516            let requests = Arc::new(Mutex::new(Vec::new()));
24517            let server_requests = Arc::clone(&requests);
24518            let thread = thread::spawn(move || {
24519                let mut request_threads = Vec::new();
24520                while !server_stop.load(Ordering::SeqCst) {
24521                    match listener.accept() {
24522                        Ok((mut stream, _)) => {
24523                            if behavior.concurrent_requests {
24524                                let requests = Arc::clone(&server_requests);
24525                                request_threads.push(thread::spawn(move || {
24526                                    handle_mock_worker_request(&mut stream, &requests, behavior)
24527                                }));
24528                            } else {
24529                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
24530                            }
24531                        }
24532                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
24533                            let mut index = 0;
24534                            while index < request_threads.len() {
24535                                if request_threads[index].is_finished() {
24536                                    request_threads
24537                                        .swap_remove(index)
24538                                        .join()
24539                                        .expect("join mock request");
24540                                } else {
24541                                    index += 1;
24542                                }
24543                            }
24544                            thread::sleep(Duration::from_millis(5));
24545                        }
24546                        Err(_) => break,
24547                    }
24548                }
24549                for request_thread in request_threads {
24550                    request_thread.join().expect("join mock request");
24551                }
24552            });
24553
24554            Self {
24555                addr,
24556                stop,
24557                requests,
24558                thread: Some(thread),
24559            }
24560        }
24561
24562        fn base_url(&self) -> String {
24563            format!("http://{}", self.addr)
24564        }
24565
24566        fn worker_protocol_for(&self, path: &str) -> Option<String> {
24567            self.requests
24568                .lock()
24569                .expect("captured requests")
24570                .iter()
24571                .find(|request| request.path == path)
24572                .and_then(|request| request.worker_protocol.clone())
24573        }
24574
24575        fn control_protocol_for(&self, path: &str) -> Option<String> {
24576            self.requests
24577                .lock()
24578                .expect("captured requests")
24579                .iter()
24580                .find(|request| request.path == path)
24581                .and_then(|request| request.control_protocol.clone())
24582        }
24583
24584        fn method_for(&self, path: &str) -> Option<String> {
24585            self.requests
24586                .lock()
24587                .expect("captured requests")
24588                .iter()
24589                .find(|request| request.path == path)
24590                .map(|request| request.method.clone())
24591        }
24592
24593        fn authorization_for(&self, path: &str) -> Option<String> {
24594            self.requests
24595                .lock()
24596                .expect("captured requests")
24597                .iter()
24598                .find(|request| request.path == path)
24599                .and_then(|request| request.authorization.clone())
24600        }
24601
24602        fn namespace_for(&self, path: &str) -> Option<String> {
24603            self.requests
24604                .lock()
24605                .expect("captured requests")
24606                .iter()
24607                .find(|request| request.path == path)
24608                .and_then(|request| request.namespace.clone())
24609        }
24610
24611        fn request_count(&self, path: &str) -> usize {
24612            self.requests
24613                .lock()
24614                .expect("captured requests")
24615                .iter()
24616                .filter(|request| request.path == path)
24617                .count()
24618        }
24619
24620        fn captured_paths(&self) -> Vec<String> {
24621            self.requests
24622                .lock()
24623                .expect("captured requests")
24624                .iter()
24625                .map(|request| request.path.clone())
24626                .collect()
24627        }
24628
24629        fn request_times(&self, path: &str) -> Vec<Instant> {
24630            self.requests
24631                .lock()
24632                .expect("captured requests")
24633                .iter()
24634                .filter(|request| request.path == path)
24635                .map(|request| request.received_at)
24636                .collect()
24637        }
24638
24639        fn request_body(&self, path: &str) -> Value {
24640            let requests = self.requests.lock().expect("captured requests");
24641            let body = &requests
24642                .iter()
24643                .find(|request| request.path == path)
24644                .unwrap_or_else(|| panic!("missing request for {path}"))
24645                .body;
24646            serde_json::from_str(body).unwrap_or_else(|error| {
24647                panic!("invalid JSON request body for {path}: {error}: {body:?}")
24648            })
24649        }
24650
24651        fn request_bodies(&self, path: &str) -> Vec<Value> {
24652            self.requests
24653                .lock()
24654                .expect("captured requests")
24655                .iter()
24656                .filter(|request| request.path == path)
24657                .map(|request| {
24658                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
24659                        panic!(
24660                            "invalid JSON request body for {path}: {error}: {:?}",
24661                            request.body
24662                        )
24663                    })
24664                })
24665                .collect()
24666        }
24667    }
24668
24669    impl Drop for MockWorkerServer {
24670        fn drop(&mut self) {
24671            self.stop.store(true, Ordering::SeqCst);
24672            let _ = TcpStream::connect(self.addr);
24673
24674            if let Some(thread) = self.thread.take() {
24675                thread.join().expect("join mock server");
24676            }
24677        }
24678    }
24679
24680    fn handle_mock_worker_request(
24681        stream: &mut TcpStream,
24682        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
24683        behavior: MockWorkerBehavior,
24684    ) {
24685        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
24686        let mut buffer = [0_u8; 8192];
24687        let mut request = Vec::new();
24688
24689        loop {
24690            match stream.read(&mut buffer) {
24691                Ok(0) => break,
24692                Ok(read) => {
24693                    request.extend_from_slice(&buffer[..read]);
24694                    if mock_request_is_complete(&request) {
24695                        break;
24696                    }
24697                }
24698                Err(error)
24699                    if matches!(
24700                        error.kind(),
24701                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
24702                    ) =>
24703                {
24704                    break;
24705                }
24706                Err(_) => return,
24707            }
24708        }
24709
24710        let request = String::from_utf8_lossy(&request);
24711        let body = request
24712            .split_once("\r\n\r\n")
24713            .map(|(_, body)| body)
24714            .unwrap_or_default();
24715        let path = request
24716            .lines()
24717            .next()
24718            .and_then(|line| line.split_whitespace().nth(1))
24719            .unwrap_or_default();
24720        let method = request
24721            .lines()
24722            .next()
24723            .and_then(|line| line.split_whitespace().next())
24724            .unwrap_or_default();
24725        let authorization = request.lines().find_map(|line| {
24726            let (name, value) = line.split_once(':')?;
24727            name.eq_ignore_ascii_case("Authorization")
24728                .then(|| value.trim().to_string())
24729        });
24730        let namespace = request.lines().find_map(|line| {
24731            let (name, value) = line.split_once(':')?;
24732            name.eq_ignore_ascii_case("X-Namespace")
24733                .then(|| value.trim().to_string())
24734        });
24735        let worker_protocol = request.lines().find_map(|line| {
24736            let (name, value) = line.split_once(':')?;
24737            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
24738                .then(|| value.trim().to_string())
24739        });
24740        let control_protocol = request.lines().find_map(|line| {
24741            let (name, value) = line.split_once(':')?;
24742            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
24743                .then(|| value.trim().to_string())
24744        });
24745        let request_number = {
24746            let mut requests = requests.lock().expect("captured requests");
24747            requests.push(CapturedRequest {
24748                method: method.to_string(),
24749                path: path.to_string(),
24750                authorization,
24751                namespace,
24752                worker_protocol: worker_protocol.clone(),
24753                control_protocol,
24754                body: body.to_string(),
24755                received_at: Instant::now(),
24756            });
24757            requests
24758                .iter()
24759                .filter(|request| request.path == path)
24760                .count()
24761        };
24762
24763        if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
24764            return;
24765        }
24766        let pressure_path = behavior
24767            .storage_path
24768            .is_some_and(|part| path.contains(part));
24769        let prior_failures = if path.ends_with("/poll") {
24770            behavior.poll_failures_per_path
24771        } else {
24772            0
24773        };
24774        if pressure_path
24775            && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
24776        {
24777            let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
24778            let poll_id = path
24779                .ends_with("/poll")
24780                .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
24781            let mut refusal = storage_refusal(
24782                poll_id,
24783                behavior.storage_unavailable,
24784                behavior.storage_mid_poll,
24785            );
24786            if behavior.storage_wrong_poll_id {
24787                refusal["poll_request_id"] = json!("wrong-poll");
24788            }
24789            write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
24790            return;
24791        }
24792        if path.contains("/storage-task/")
24793            || path.contains("/storage-activity/")
24794            || path.contains("/storage-query/")
24795        {
24796            write_mock_response(stream, "200 OK", "{}");
24797            return;
24798        }
24799        if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
24800            write_mock_response(stream, "200 OK", &json!({"task":{
24801                "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
24802                "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
24803                "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24804                "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24805                "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
24806            }}).to_string());
24807            return;
24808        }
24809        if behavior.storage_activity
24810            && path == "/api/worker/activity-tasks/poll"
24811            && request_number == 1
24812        {
24813            write_mock_response(stream, "200 OK", &json!({"task":{
24814                "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
24815                "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24816                "attempt_number":7, "lease_owner":"storage-worker"
24817            }}).to_string());
24818            return;
24819        }
24820
24821        if path == "/api/worker/register" {
24822            if behavior.reject_registration_protocol {
24823                write_mock_response(
24824                    stream,
24825                    "400 Bad Request",
24826                    r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
24827                );
24828                return;
24829            }
24830            if behavior.reject_registration {
24831                write_mock_response(
24832                    stream,
24833                    "503 Service Unavailable",
24834                    r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
24835                );
24836                return;
24837            }
24838        }
24839
24840        if path.starts_with("/api/worker/registrations/") {
24841            if behavior.reject_deregistration_protocol {
24842                write_mock_response(
24843                    stream,
24844                    "400 Bad Request",
24845                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
24846                );
24847            } else if behavior.reject_deregistration {
24848                write_mock_response(
24849                    stream,
24850                    "403 Forbidden",
24851                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
24852                );
24853            } else {
24854                write_mock_response(
24855                    stream,
24856                    "200 OK",
24857                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
24858                );
24859            }
24860            return;
24861        }
24862
24863        let is_poll = matches!(
24864            path,
24865            "/api/worker/workflow-tasks/poll"
24866                | "/api/worker/activity-tasks/poll"
24867                | "/api/worker/query-tasks/poll"
24868        );
24869        if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
24870            write_mock_response(
24871                stream,
24872                "429 Too Many Requests",
24873                r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
24874            );
24875            return;
24876        }
24877        if is_poll && request_number <= behavior.poll_failures_per_path {
24878            return;
24879        }
24880        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
24881            return;
24882        }
24883        if path == "/api/worker/heartbeat"
24884            && behavior.heartbeat_failure_request == Some(request_number)
24885        {
24886            return;
24887        }
24888        if path == "/api/worker/heartbeat"
24889            && behavior.delayed_heartbeat_request == Some(request_number)
24890        {
24891            thread::sleep(behavior.heartbeat_response_delay);
24892        }
24893        if behavior.unauthorized_polls && is_poll {
24894            write_mock_response(
24895                stream,
24896                "401 Unauthorized",
24897                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
24898            );
24899            return;
24900        }
24901        if behavior.draining_polls && is_poll {
24902            write_mock_response(
24903                stream,
24904                "409 Conflict",
24905                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
24906            );
24907            return;
24908        }
24909
24910        if let Some(codec_case) = behavior.invalid_task_payload_codec {
24911            if is_poll && request_number == 1 {
24912                let mut task = match path {
24913                    "/api/worker/workflow-tasks/poll" => json!({
24914                        "task_id": "codec-workflow",
24915                        "workflow_type": "codec.workflow",
24916                        "payload_codec": DEFAULT_CODEC,
24917                        "workflow_task_attempt": 1,
24918                        "lease_owner": "codec-worker"
24919                    }),
24920                    "/api/worker/activity-tasks/poll" => json!({
24921                        "task_id": "codec-activity",
24922                        "activity_attempt_id": "codec-activity-attempt",
24923                        "activity_type": "codec.activity",
24924                        "payload_codec": DEFAULT_CODEC,
24925                        "attempt_number": 1,
24926                        "lease_owner": "codec-worker"
24927                    }),
24928                    "/api/worker/query-tasks/poll" => json!({
24929                        "query_task_id": "codec-query",
24930                        "query_task_attempt": 1,
24931                        "workflow_type": "codec.workflow",
24932                        "query_name": "known",
24933                        "payload_codec": DEFAULT_CODEC,
24934                        "lease_owner": "codec-worker"
24935                    }),
24936                    _ => unreachable!("is_poll limits task codec probe paths"),
24937                };
24938                codec_case.apply(&mut task);
24939                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
24940                return;
24941            }
24942
24943            if matches!(
24944                path,
24945                "/api/worker/workflow-tasks/codec-workflow/fail"
24946                    | "/api/worker/activity-tasks/codec-activity/fail"
24947                    | "/api/worker/query-tasks/codec-query/fail"
24948            ) {
24949                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
24950                return;
24951            }
24952        }
24953
24954        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
24955            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
24956            let body = format!(
24957                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
24958            );
24959            write_mock_response(stream, "400 Bad Request", &body);
24960            return;
24961        }
24962
24963        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
24964        {
24965            write_mock_response(
24966                stream,
24967                "409 Conflict",
24968                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
24969            );
24970            return;
24971        }
24972
24973        if behavior.workflow_completion_status.is_some()
24974            && path == "/api/worker/workflow-tasks/poll"
24975            && request_number == 1
24976        {
24977            write_mock_response(
24978                stream,
24979                "200 OK",
24980                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
24981            );
24982            return;
24983        }
24984
24985        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
24986            if let (Some(status), Some(body)) = (
24987                behavior.workflow_completion_status,
24988                behavior.workflow_completion_body,
24989            ) {
24990                write_mock_response(stream, status, body);
24991                return;
24992            }
24993        }
24994
24995        if behavior.waiting_query_worker {
24996            if behavior.complete_named_signal
24997                && path == "/api/worker/workflow-tasks/poll"
24998                && request_number == 1
24999            {
25000                let body = json!({
25001                    "task": {
25002                        "task_id": "snapshot-open",
25003                        "workflow_id": "snapshot-1",
25004                        "run_id": "snapshot-run-1",
25005                        "workflow_type": "snapshot",
25006                        "payload_codec": DEFAULT_CODEC,
25007                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25008                            .expect("Avro workflow arguments"),
25009                        "history_events": [],
25010                        "workflow_task_attempt": 1,
25011                        "lease_owner": "rust-snapshot-worker"
25012                    }
25013                })
25014                .to_string();
25015                write_mock_response(stream, "200 OK", &body);
25016                return;
25017            }
25018
25019            let signal_request = request_number - usize::from(behavior.complete_named_signal);
25020            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25021            if path == "/api/worker/workflow-tasks/poll"
25022                && signal_request >= 1
25023                && signal_request <= signal_request_limit
25024            {
25025                let finish = behavior.complete_named_signal && signal_request == 3;
25026                let amounts = if signal_request == 1 {
25027                    vec![3]
25028                } else {
25029                    vec![3, 5]
25030                };
25031                let task_id = if signal_request == 1 {
25032                    "snapshot-wait-3"
25033                } else if finish {
25034                    "snapshot-finish"
25035                } else {
25036                    "snapshot-wait-5"
25037                };
25038                let mut history_events = std::iter::once(json!({
25039                    "event_type": "SignalWaitOpened",
25040                    "payload": {"sequence": 1, "signal_name": "finish"}
25041                }))
25042                .chain(amounts.iter().enumerate().map(|(index, amount)| {
25043                    json!({
25044                        "event_type": "SignalReceived",
25045                        "payload": {
25046                            "signal_id": format!("increment-{amount}"),
25047                            "signal_name": "increment",
25048                            "workflow_sequence": index + 2,
25049                            "payload_codec": DEFAULT_CODEC,
25050                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25051                                .expect("Avro signal envelope")
25052                        }
25053                    })
25054                }))
25055                .collect::<Vec<_>>();
25056                let (resume_id, resume_name, resume_arguments) = if finish {
25057                    history_events.push(json!({
25058                        "event_type": "SignalReceived",
25059                        "payload": {
25060                            "signal_id": "finish",
25061                            "signal_name": "finish",
25062                            "workflow_sequence": 4,
25063                            "payload_codec": DEFAULT_CODEC,
25064                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25065                                .expect("Avro finish signal envelope")
25066                        }
25067                    }));
25068                    (
25069                        "finish".to_string(),
25070                        "finish".to_string(),
25071                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
25072                            .expect("Avro finish resume signal"),
25073                    )
25074                } else {
25075                    let amount = amounts.last().expect("amount");
25076                    (
25077                        format!("increment-{amount}"),
25078                        "increment".to_string(),
25079                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25080                            .expect("Avro increment resume signal"),
25081                    )
25082                };
25083                let body = json!({
25084                    "task": {
25085                        "task_id": task_id,
25086                        "workflow_id": "snapshot-1",
25087                        "run_id": "snapshot-run-1",
25088                        "workflow_type": "snapshot",
25089                        "payload_codec": DEFAULT_CODEC,
25090                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25091                            .expect("Avro workflow arguments"),
25092                        "history_events": history_events,
25093                        "workflow_task_attempt": 1,
25094                        "workflow_signal_id": resume_id,
25095                        "signal_name": resume_name,
25096                        "signal_arguments": resume_arguments,
25097                        "lease_owner": "rust-snapshot-worker"
25098                    }
25099                })
25100                .to_string();
25101                write_mock_response(stream, "200 OK", &body);
25102                return;
25103            }
25104
25105            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25106                let history_events = [3, 5]
25107                    .into_iter()
25108                    .enumerate()
25109                    .map(|(index, amount)| {
25110                        json!({
25111                            "event_type": "SignalReceived",
25112                            "payload": {
25113                                "signal_id": format!("increment-{amount}"),
25114                                "signal_name": "increment",
25115                                "workflow_sequence": index + 2,
25116                                "payload_codec": DEFAULT_CODEC,
25117                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25118                                    .expect("Avro query signal envelope")
25119                            }
25120                        })
25121                    })
25122                    .collect::<Vec<_>>();
25123                let body = json!({
25124                    "task": {
25125                        "query_task_id": "snapshot-current",
25126                        "query_task_attempt": 1,
25127                        "lease_owner": "rust-snapshot-worker",
25128                        "workflow_id": "snapshot-1",
25129                        "run_id": "snapshot-run-1",
25130                        "workflow_type": "snapshot",
25131                        "query_name": "current",
25132                        "payload_codec": DEFAULT_CODEC,
25133                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25134                            .expect("Avro workflow arguments"),
25135                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25136                            .expect("Avro query arguments"),
25137                        "history_events": history_events,
25138                        "run_status": "waiting"
25139                    }
25140                })
25141                .to_string();
25142                write_mock_response(stream, "200 OK", &body);
25143                return;
25144            }
25145
25146            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25147                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25148            {
25149                write_mock_response(
25150                    stream,
25151                    "200 OK",
25152                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
25153                );
25154                return;
25155            }
25156
25157            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25158                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25159                return;
25160            }
25161
25162            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25163                write_mock_response(
25164                    stream,
25165                    "200 OK",
25166                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25167                );
25168                return;
25169            }
25170
25171            if path == "/api/worker/query-tasks/snapshot-current/complete" {
25172                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
25173                return;
25174            }
25175        }
25176
25177        if matches!(
25178            path,
25179            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
25180        ) {
25181            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25182                .expect("typed mock result");
25183            let body = json!({
25184                "result": typed_fidelity_probe().into_json().expect("result projection"),
25185                "result_envelope": result,
25186            })
25187            .to_string();
25188            write_mock_response(stream, "200 OK", &body);
25189            return;
25190        }
25191
25192        if path == "/api/workflows/typed-1" {
25193            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25194                .expect("typed mock result");
25195            let body = json!({
25196                "workflow_id": "typed-1",
25197                "run_id": "run-typed-1",
25198                "workflow_type": "typed.echo",
25199                "status": "completed",
25200                "output": typed_fidelity_probe().into_json().expect("output projection"),
25201                "output_envelope": result,
25202            })
25203            .to_string();
25204            write_mock_response(stream, "200 OK", &body);
25205            return;
25206        }
25207
25208        let (status, body) = match path {
25209            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
25210            "/api/workflows" => (
25211                "201 Created",
25212                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
25213            ),
25214            "/api/worker/register" if behavior.decline_registration => (
25215                "200 OK",
25216                r#"{"worker_id":"declined-worker","registered":false}"#,
25217            ),
25218            "/api/worker/register" if behavior.waiting_query_worker => (
25219                "200 OK",
25220                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
25221            ),
25222            "/api/worker/register" => (
25223                "200 OK",
25224                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
25225            ),
25226            "/api/worker/heartbeat" => ("200 OK", "{}"),
25227            "/api/worker/activity-tasks/poll"
25228                if behavior.cancelled_activity && request_number == 1 =>
25229            {
25230                (
25231                    "200 OK",
25232                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
25233                )
25234            }
25235            "/api/worker/activity-tasks/poll"
25236                if behavior.long_poll_capacity_responses_per_path > 0
25237                    && request_number
25238                        == behavior
25239                            .long_poll_capacity_responses_per_path
25240                            .saturating_add(1) =>
25241            {
25242                (
25243                    "200 OK",
25244                    r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
25245                )
25246            }
25247            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
25248                ("200 OK", r#"{"task":null}"#)
25249            }
25250            "/api/worker/query-tasks/poll"
25251                if behavior.reject_query_completion && request_number == 1 =>
25252            {
25253                (
25254                    "200 OK",
25255                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
25256                )
25257            }
25258            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
25259            "/api/worker/query-tasks/query-capture/complete"
25260            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
25261            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
25262                "200 OK",
25263                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
25264            ),
25265            "/api/worker/activity-tasks/activity-cancel/complete" => (
25266                "409 Conflict",
25267                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
25268            ),
25269            "/api/worker/activity-tasks/activity-typed/complete"
25270            | "/api/worker/activity-tasks/activity-typed/fail"
25271            | "/api/worker/activity-tasks/capacity-activity/complete"
25272            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
25273            "/api/workflows/counter-1/query/current" => (
25274                "200 OK",
25275                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
25276            ),
25277            "/api/workflows/counter-1/query/missing" => (
25278                "404 Not Found",
25279                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
25280            ),
25281            "/api/workflows/wf-lifecycle/cancel" => (
25282                "200 OK",
25283                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
25284            ),
25285            "/api/workflows/wf-lifecycle/terminate" => (
25286                "200 OK",
25287                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
25288            ),
25289            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
25290                "200 OK",
25291                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
25292            ),
25293            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
25294                "200 OK",
25295                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
25296            ),
25297            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
25298            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
25299                "409 Conflict",
25300                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
25301            ),
25302            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
25303                "200 OK",
25304                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
25305            ),
25306            "/api/workflows/wf-cancelled" => (
25307                "200 OK",
25308                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
25309            ),
25310            "/api/workflows/wf-terminated" => (
25311                "200 OK",
25312                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
25313            ),
25314            "/api/workflows/wf-timed-out" => (
25315                "200 OK",
25316                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
25317            ),
25318            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
25319                "200 OK",
25320                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
25321            ),
25322            "/api/workflows/wf-selected" => (
25323                "200 OK",
25324                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
25325            ),
25326            "/api/workflows/wf-selected/runs/run-selected" => (
25327                "200 OK",
25328                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
25329            ),
25330            _ => ("404 Not Found", r#"{"message":"not found"}"#),
25331        };
25332        write_mock_response(stream, status, body);
25333    }
25334
25335    fn mock_request_is_complete(request: &[u8]) -> bool {
25336        let Some(header_end) = request
25337            .windows(4)
25338            .position(|window| window == b"\r\n\r\n")
25339            .map(|position| position + 4)
25340        else {
25341            return false;
25342        };
25343        let headers = String::from_utf8_lossy(&request[..header_end]);
25344        let content_length = headers.lines().find_map(|line| {
25345            let (name, value) = line.split_once(':')?;
25346            name.eq_ignore_ascii_case("content-length")
25347                .then(|| value.trim().parse::<usize>().ok())
25348                .flatten()
25349        });
25350
25351        request.len() >= header_end + content_length.unwrap_or(0)
25352    }
25353
25354    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
25355        let response = format!(
25356            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
25357            body.len()
25358        );
25359
25360        let _ = stream.write_all(response.as_bytes());
25361        let _ = stream.flush();
25362    }
25363}