Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{type_name, Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use chrono::DateTime;
20use futures_util::{future::OptionFuture, task::noop_waker_ref};
21use serde::{
22    de::DeserializeOwned,
23    ser::{SerializeMap, SerializeSeq},
24    Deserialize, Deserializer, Serialize, Serializer,
25};
26pub use serde_json::{json, Value};
27use sha2::{Digest, Sha256};
28use thiserror::Error;
29pub use uuid::Uuid;
30
31pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
32/// First additive worker protocol that defines portable worker-affinity features.
33pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
34pub const CONTROL_PLANE_VERSION: &str = "2";
35pub const DEFAULT_CODEC: &str = "avro";
36pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
37/// Worker-registration capability for authored condition-wait occurrence identity.
38pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
39    "condition_wait_occurrence_identity";
40/// Worker-registration capability for portable memo upserts.
41pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
42/// Worker-registration capability for server-routed read-only queries.
43pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
44/// Worker-registration capability for canonical typed search attributes.
45pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
46/// Worker-registration capability for synchronous workflow updates.
47pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
48/// Worker-registration capability for durable named input streams.
49pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
50/// Worker-registration capability for persisted first-completion selection.
51pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
52pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
53pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
54pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
55pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
56pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
57/// First additive worker protocol that defines query-task transport.
58pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
59/// First additive worker protocol that defines typed search-attribute upserts.
60pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
61/// First additive worker protocol that defines portable memo upserts.
62pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
63/// First additive worker protocol that preserves declared search-attribute types.
64pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
65/// First additive worker protocol that defines external durable condition waits.
66pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
67/// First additive worker protocol that preserves authored condition-wait occurrences.
68pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
69/// First additive worker protocol that defines durable selection groups.
70pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
71
72pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
73    let Some((major, minor)) = version.split_once('.') else {
74        return false;
75    };
76    major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
77}
78
79fn validate_user_signal_name(signal_name: &str) -> Result<()> {
80    if signal_name == MESSAGE_STREAM_SIGNAL {
81        return Err(Error::Codec(format!(
82            "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
83        )));
84    }
85    Ok(())
86}
87
88const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
89const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
90    "Workflow task waiting for scheduled history.";
91const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
92const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
93const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
94const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
95const MAX_MEMO_ENTRIES: usize = 100;
96const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
97const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
98
99const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
100    "lease_expired",
101    "query_task_not_found",
102    "query_task_not_leased",
103    "query_task_timed_out",
104];
105
106/// Truthful service-worker manifest for features this SDK currently refuses.
107pub fn portable_worker_affinity_capability_manifest() -> Value {
108    json!({
109        "local_activities": {
110            "supported": false,
111            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
112            "reason": "rust_worker_does_not_execute_record_local_activity",
113        },
114        "worker_sessions": {
115            "supported": false,
116            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
117            "reason": "rust_worker_has_no_typed_session_lifecycle",
118        },
119        "sticky_execution": {
120            "supported": false,
121            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
122            "reason": "rust_worker_uses_complete_durable_history_replay",
123        },
124    })
125}
126
127/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
128pub const AVRO_VALUE_SCHEMA_JSON: &str =
129    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
130pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
131pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
132const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
133
134static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
135static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
136    OnceLock::new();
137
138#[derive(Clone, Copy)]
139enum RequestProtocol {
140    ControlPlane,
141    Worker(&'static str),
142}
143
144pub type Result<T> = std::result::Result<T, Error>;
145
146#[derive(Debug, Error)]
147pub enum Error {
148    #[error("transport error: {0}")]
149    Transport(#[from] reqwest::Error),
150    #[error(
151        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
152    )]
153    InvalidBaseUrl,
154    #[error("json error: {0}")]
155    Json(#[from] serde_json::Error),
156    #[error("http {status}: {body}")]
157    Http {
158        status: reqwest::StatusCode,
159        body: String,
160    },
161    #[error("codec error: {0}")]
162    Codec(String),
163    #[error(transparent)]
164    QueryFailed(QueryFailure),
165    #[error(transparent)]
166    Protocol(ProtocolFailure),
167    #[error(transparent)]
168    NonDeterministicReplay(ReplayFailure),
169    #[error(transparent)]
170    ChildWorkflowFailed(ChildWorkflowFailure),
171    #[error(transparent)]
172    ActivityFailed(ActivityFailure),
173    #[error(transparent)]
174    ParallelFailed(ParallelFailure),
175    #[error(transparent)]
176    SagaCompensationFailed(SagaCompensationFailure),
177    #[error(transparent)]
178    InvalidParallelGroup(ParallelGroupError),
179    #[error(transparent)]
180    DurableOperationCancelled(DurableOperationCancelled),
181    #[error(transparent)]
182    WorkflowCancellationRequested(WorkflowCancellationRequested),
183    #[error(transparent)]
184    WorkflowCommandRejected(WorkflowCommandRejection),
185    #[error(transparent)]
186    WorkflowFailed(WorkflowTerminalOutcome),
187    #[error(transparent)]
188    WorkflowCancelled(WorkflowTerminalOutcome),
189    #[error(transparent)]
190    WorkflowTerminated(WorkflowTerminalOutcome),
191    #[error(transparent)]
192    WorkflowTimedOut(WorkflowTerminalOutcome),
193    #[error(transparent)]
194    ActivityTaskRejected(ActivityTaskRejection),
195    #[error("workflow handler {0:?} is not registered")]
196    WorkflowNotRegistered(String),
197    #[error("activity handler {0:?} is not registered")]
198    ActivityNotRegistered(String),
199    #[error(
200        "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
201    )]
202    HandlerType {
203        handler_kind: HandlerKind,
204        handler_name: String,
205        value_kind: HandlerValueKind,
206        rust_type: &'static str,
207        message: String,
208    },
209    #[error("workflow future yielded without emitting a durable command")]
210    WorkflowYieldedWithoutCommand,
211    #[error(
212        "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
213    )]
214    MissingWorkflowCommandIdentity,
215    #[error("workflow state lock is poisoned")]
216    WorkflowStatePoisoned,
217    #[error("timer duration is too large for the worker protocol")]
218    TimerDurationOverflow,
219    #[error(transparent)]
220    InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
221    #[error(transparent)]
222    InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
223    #[error("operation timed out")]
224    Timeout,
225    #[error(
226        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
227    )]
228    MissingRoleCredentials {
229        role: &'static str,
230        opposite_role: &'static str,
231    },
232    #[error("worker loop error: {0}")]
233    WorkerLoop(String),
234    #[error(
235        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
236    )]
237    UnsupportedUpdateValidators { workflow_type: String },
238    #[error("{primary}; worker deregistration also failed: {deregistration}")]
239    WorkerShutdown {
240        primary: Box<Error>,
241        deregistration: Box<Error>,
242    },
243    #[error("invalid child workflow options: {0}")]
244    InvalidChildWorkflowOptions(String),
245    #[error("invalid workflow memo update: {0}")]
246    InvalidMemoUpdate(String),
247    #[error(
248        "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
249    )]
250    WorkflowMemoUpdatesUnavailable,
251    #[error(transparent)]
252    InvalidActivityOptions(ActivityOptionsError),
253    #[error(transparent)]
254    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
255    #[doc(hidden)]
256    #[error("workflow requested continue as new")]
257    ContinueAsNew(ContinueAsNewRequest),
258}
259
260/// Validation failure for a durable condition-wait definition.
261#[derive(Clone, Debug, Error, PartialEq, Eq)]
262pub enum ConditionWaitOptionsError {
263    #[error("condition_key must be non-empty")]
264    EmptyKey,
265    #[error("condition_definition_fingerprint must be non-empty")]
266    EmptyPredicateIdentity,
267    #[error("condition timeout is too large for the worker protocol")]
268    TimeoutOverflow,
269}
270
271/// Stable identity and optional durable timeout for a condition wait.
272///
273/// `predicate_identity` is recorded as the worker protocol's
274/// `condition_definition_fingerprint` and must change whenever predicate
275/// behavior changes. Prefer the [`wait_condition!`] macro when the predicate
276/// is written inline; it derives this identity from the predicate tokens.
277#[derive(Clone, Debug, PartialEq, Eq)]
278pub struct ConditionWaitOptions {
279    condition_key: String,
280    predicate_identity: String,
281    timeout: Option<Duration>,
282}
283
284impl ConditionWaitOptions {
285    pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
286        Self {
287            condition_key: condition_key.into(),
288            predicate_identity: predicate_identity.into(),
289            timeout: None,
290        }
291    }
292
293    pub fn timeout(mut self, timeout: Duration) -> Self {
294        self.timeout = Some(timeout);
295        self
296    }
297
298    fn validate(
299        &self,
300    ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
301        let condition_key = self.condition_key.trim();
302        if condition_key.is_empty() {
303            return Err(ConditionWaitOptionsError::EmptyKey);
304        }
305        let predicate_identity = self.predicate_identity.trim();
306        if predicate_identity.is_empty() {
307            return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
308        }
309        let timeout_seconds = self
310            .timeout
311            .map(|timeout| {
312                timeout
313                    .as_secs()
314                    .checked_add(u64::from(timeout.subsec_nanos() > 0))
315                    .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
316            })
317            .transpose()?;
318
319        Ok(ValidatedConditionWaitOptions {
320            condition_key: condition_key.to_string(),
321            predicate_identity: predicate_identity.to_string(),
322            timeout_seconds,
323        })
324    }
325}
326
327#[derive(Clone, Debug, PartialEq, Eq)]
328struct ValidatedConditionWaitOptions {
329    condition_key: String,
330    predicate_identity: String,
331    timeout_seconds: Option<u64>,
332}
333
334const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
335
336/// Unambiguous terminal result of a durable condition wait.
337#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
338#[serde(rename_all = "snake_case")]
339pub enum ConditionWaitResult {
340    Satisfied,
341    TimedOut,
342}
343
344impl ConditionWaitResult {
345    pub fn is_satisfied(self) -> bool {
346        self == Self::Satisfied
347    }
348
349    pub fn is_timed_out(self) -> bool {
350        self == Self::TimedOut
351    }
352}
353
354/// Build the stable condition definition identity used by [`wait_condition!`].
355#[doc(hidden)]
356pub fn __condition_definition_fingerprint(source: &str) -> String {
357    let mut digest = Sha256::new();
358    digest.update(b"durable-workflow-rust.wait-condition.v1\0");
359    digest.update(source.as_bytes());
360    format!("sha256:{:x}", digest.finalize())
361}
362
363/// Create a durable condition wait whose predicate definition is fingerprinted
364/// from its inline Rust tokens.
365///
366/// The returned [`ConditionWaitCall`] must be awaited. The timeout form is
367/// `wait_condition!(ctx, "approval", timeout: duration, || predicate)`.
368#[macro_export]
369macro_rules! wait_condition {
370    ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
371        $ctx.wait_condition(
372            $crate::ConditionWaitOptions::new(
373                $key,
374                $crate::__condition_definition_fingerprint(concat!(
375                    module_path!(),
376                    "\0",
377                    stringify!($predicate)
378                )),
379            )
380            .timeout($timeout),
381            $predicate,
382        )
383    }};
384    ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
385        $ctx.wait_condition(
386            $crate::ConditionWaitOptions::new(
387                $key,
388                $crate::__condition_definition_fingerprint(concat!(
389                    module_path!(),
390                    "\0",
391                    stringify!($predicate)
392                )),
393            ),
394            $predicate,
395        )
396    }};
397}
398
399const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
400const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
401const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
402const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
403const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
404
405/// Validation failure for a typed workflow search-attribute update.
406#[derive(Clone, Debug, Error, PartialEq, Eq)]
407pub enum SearchAttributeUpdateError {
408    #[error("search-attribute update requires at least one attribute")]
409    Empty,
410    #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
411    InvalidKey(String),
412    #[error("search-attribute update exceeds the limit of 100 attributes")]
413    TooManyAttributes,
414    #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
415    ValueTooLong {
416        key: String,
417        kind: &'static str,
418        limit: usize,
419    },
420    #[error(
421        "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
422    )]
423    EmptyString(String),
424    #[error("search attribute {0:?} has a non-finite float value")]
425    NonFiniteFloat(String),
426    #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
427    InvalidDateTime(String),
428    #[error("search-attribute update exceeds the 65536-byte protocol limit")]
429    PayloadTooLarge,
430}
431
432/// One public typed search-attribute value.
433#[derive(Clone, Debug, PartialEq)]
434pub enum SearchAttributeValue {
435    String(String),
436    Keyword(String),
437    KeywordList(Vec<String>),
438    Int(i64),
439    Float(f64),
440    Bool(bool),
441    DateTime(String),
442    Delete,
443}
444
445impl SearchAttributeValue {
446    fn type_name(&self) -> Option<&'static str> {
447        match self {
448            Self::String(_) => Some("string"),
449            Self::Keyword(_) => Some("keyword"),
450            Self::KeywordList(_) => Some("keyword_list"),
451            Self::Int(_) => Some("int"),
452            Self::Float(_) => Some("float"),
453            Self::Bool(_) => Some("bool"),
454            Self::DateTime(_) => Some("datetime"),
455            Self::Delete => None,
456        }
457    }
458
459    fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
460        let normalize_string = |value: String, kind: &'static str, limit: usize| {
461            let value = value.trim().to_string();
462            if value.is_empty() {
463                return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
464            }
465            if value.len() > limit {
466                return Err(SearchAttributeUpdateError::ValueTooLong {
467                    key: key.to_string(),
468                    kind,
469                    limit,
470                });
471            }
472            Ok(value)
473        };
474
475        match self {
476            Self::String(value) => Ok(Self::String(normalize_string(
477                value,
478                "string",
479                MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
480            )?)),
481            Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
482                value,
483                "keyword",
484                MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
485            )?)),
486            Self::KeywordList(values) => {
487                let values = values
488                    .into_iter()
489                    .map(|value| {
490                        let value = value.trim().to_string();
491                        if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
492                            return Err(SearchAttributeUpdateError::ValueTooLong {
493                                key: key.to_string(),
494                                kind: "keyword-list entry",
495                                limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
496                            });
497                        }
498                        Ok(value)
499                    })
500                    .collect::<std::result::Result<Vec<_>, _>>()?;
501                Ok(Self::KeywordList(values))
502            }
503            Self::Float(value) if !value.is_finite() => {
504                Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
505            }
506            Self::DateTime(value) => {
507                let value =
508                    normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
509                if DateTime::parse_from_rfc3339(&value).is_err() {
510                    return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
511                }
512                Ok(Self::DateTime(value))
513            }
514            value => Ok(value),
515        }
516    }
517
518    fn into_json(self) -> Value {
519        match self {
520            Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
521                Value::String(value)
522            }
523            Self::KeywordList(values) => {
524                Value::Array(values.into_iter().map(Value::String).collect())
525            }
526            Self::Int(value) => json!(value),
527            Self::Float(value) => json!(value),
528            Self::Bool(value) => json!(value),
529            Self::Delete => Value::Null,
530        }
531    }
532}
533
534/// Validated typed workflow-side search-attribute mutation.
535#[derive(Clone, Debug, Default, PartialEq)]
536pub struct SearchAttributeUpdate {
537    attributes: BTreeMap<String, SearchAttributeValue>,
538}
539
540impl SearchAttributeUpdate {
541    pub fn new() -> Self {
542        Self::default()
543    }
544
545    pub fn set(
546        mut self,
547        key: impl Into<String>,
548        value: SearchAttributeValue,
549    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
550        let key = key.into();
551        validate_search_attribute_key(&key)?;
552        if !self.attributes.contains_key(&key)
553            && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
554        {
555            return Err(SearchAttributeUpdateError::TooManyAttributes);
556        }
557        self.attributes.insert(key.clone(), value.normalized(&key)?);
558        self.validate_size()?;
559        Ok(self)
560    }
561
562    pub fn string(
563        self,
564        key: impl Into<String>,
565        value: impl Into<String>,
566    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
567        self.set(key, SearchAttributeValue::String(value.into()))
568    }
569
570    pub fn keyword(
571        self,
572        key: impl Into<String>,
573        value: impl Into<String>,
574    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
575        self.set(key, SearchAttributeValue::Keyword(value.into()))
576    }
577
578    pub fn keyword_list<I, V>(
579        self,
580        key: impl Into<String>,
581        values: I,
582    ) -> std::result::Result<Self, SearchAttributeUpdateError>
583    where
584        I: IntoIterator<Item = V>,
585        V: Into<String>,
586    {
587        self.set(
588            key,
589            SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
590        )
591    }
592
593    pub fn int(
594        self,
595        key: impl Into<String>,
596        value: i64,
597    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
598        self.set(key, SearchAttributeValue::Int(value))
599    }
600
601    pub fn float(
602        self,
603        key: impl Into<String>,
604        value: f64,
605    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
606        self.set(key, SearchAttributeValue::Float(value))
607    }
608
609    pub fn bool(
610        self,
611        key: impl Into<String>,
612        value: bool,
613    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
614        self.set(key, SearchAttributeValue::Bool(value))
615    }
616
617    pub fn datetime(
618        self,
619        key: impl Into<String>,
620        value: impl Into<String>,
621    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
622        self.set(key, SearchAttributeValue::DateTime(value.into()))
623    }
624
625    pub fn delete(
626        self,
627        key: impl Into<String>,
628    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
629        self.set(key, SearchAttributeValue::Delete)
630    }
631
632    fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
633        let (attributes, _) = self.clone().into_wire_parts();
634        if serde_json::to_vec(&attributes)
635            .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
636            .unwrap_or(true)
637        {
638            return Err(SearchAttributeUpdateError::PayloadTooLarge);
639        }
640        Ok(())
641    }
642
643    fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
644        let mut attributes = serde_json::Map::new();
645        let mut attribute_types = BTreeMap::new();
646        for (key, value) in self.attributes {
647            if let Some(type_name) = value.type_name() {
648                attribute_types.insert(key.clone(), type_name.to_string());
649            }
650            attributes.insert(key, value.into_json());
651        }
652        (Value::Object(attributes), attribute_types)
653    }
654
655    fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
656        if self.attributes.is_empty() {
657            return Err(SearchAttributeUpdateError::Empty);
658        }
659        self.validate_size()
660    }
661}
662
663fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
664    let valid = !key.is_empty()
665        && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
666        && key
667            .bytes()
668            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
669    if valid {
670        Ok(())
671    } else {
672        Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
673    }
674}
675
676/// The registered handler family reported by [`Error::HandlerType`].
677#[derive(Clone, Copy, Debug, PartialEq, Eq)]
678pub enum HandlerKind {
679    Workflow,
680    Activity,
681}
682
683impl std::fmt::Display for HandlerKind {
684    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
685        formatter.write_str(match self {
686            Self::Workflow => "workflow",
687            Self::Activity => "activity",
688        })
689    }
690}
691
692/// Whether a typed handler failed to adapt its input or result.
693#[derive(Clone, Copy, Debug, PartialEq, Eq)]
694pub enum HandlerValueKind {
695    Input,
696    Result,
697}
698
699impl std::fmt::Display for HandlerValueKind {
700    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
701        formatter.write_str(match self {
702            Self::Input => "input",
703            Self::Result => "result",
704        })
705    }
706}
707
708/// The lifecycle command sent to a workflow execution.
709#[derive(Clone, Copy, Debug, PartialEq, Eq)]
710pub enum WorkflowCommandKind {
711    Cancel,
712    Terminate,
713}
714
715impl WorkflowCommandKind {
716    fn as_str(self) -> &'static str {
717        match self {
718            Self::Cancel => "cancel",
719            Self::Terminate => "terminate",
720        }
721    }
722}
723
724/// Optional structured fields for a cancellation or termination request.
725#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
726pub struct WorkflowCommandOptions {
727    #[serde(skip_serializing_if = "Option::is_none")]
728    pub reason: Option<String>,
729    #[serde(skip_serializing_if = "Option::is_none")]
730    pub request_id: Option<String>,
731}
732
733/// Server-enforced timeout policy for a workflow start.
734///
735/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
736/// only bounds how long the caller waits. A server deadline produces a terminal
737/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
738/// `run_timeout`.
739#[derive(Clone, Debug, PartialEq, Eq)]
740pub struct WorkflowStartOptions {
741    pub execution_timeout_seconds: u64,
742    pub run_timeout_seconds: u64,
743}
744
745impl Default for WorkflowStartOptions {
746    fn default() -> Self {
747        Self {
748            execution_timeout_seconds: 3600,
749            run_timeout_seconds: 600,
750        }
751    }
752}
753
754impl WorkflowStartOptions {
755    pub fn new() -> Self {
756        Self::default()
757    }
758
759    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
760        self.execution_timeout_seconds = seconds;
761        self
762    }
763
764    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
765        self.run_timeout_seconds = seconds;
766        self
767    }
768
769    fn validate(&self) -> Result<()> {
770        if self.execution_timeout_seconds == 0 {
771            return Err(Error::Codec(
772                "execution_timeout_seconds must be at least 1".to_string(),
773            ));
774        }
775        if self.run_timeout_seconds == 0 {
776            return Err(Error::Codec(
777                "run_timeout_seconds must be at least 1".to_string(),
778            ));
779        }
780        if self.run_timeout_seconds > self.execution_timeout_seconds {
781            return Err(Error::Codec(
782                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
783            ));
784        }
785
786        Ok(())
787    }
788}
789
790/// Optional routing overrides for a continue-as-new transition.
791///
792/// Omitted values retain the current workflow type and task queue. Server-owned
793/// instance metadata is not accepted here and is carried by the server.
794#[derive(Clone, Debug, Default, PartialEq, Eq)]
795pub struct ContinueAsNewOptions {
796    pub workflow_type: Option<String>,
797    pub task_queue: Option<String>,
798}
799
800impl ContinueAsNewOptions {
801    pub fn new() -> Self {
802        Self::default()
803    }
804
805    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
806        self.workflow_type = Some(workflow_type.into());
807        self
808    }
809
810    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
811        self.task_queue = Some(task_queue.into());
812        self
813    }
814
815    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
816        for (field, value) in [
817            ("workflow_type", self.workflow_type.as_deref()),
818            ("task_queue", self.task_queue.as_deref()),
819        ] {
820            if value.is_some_and(|value| value.trim().is_empty()) {
821                return Err(ContinueAsNewOptionsError {
822                    field,
823                    message: format!("{field} must not be empty"),
824                });
825            }
826        }
827        Ok(())
828    }
829}
830
831/// A stable validation error raised before a continue-as-new command is emitted.
832#[derive(Clone, Debug, Error, PartialEq, Eq)]
833#[error("invalid continue-as-new option {field}: {message}")]
834pub struct ContinueAsNewOptionsError {
835    pub field: &'static str,
836    pub message: String,
837}
838
839/// Public history-budget information attached to the current workflow task.
840#[derive(Clone, Debug, Default, PartialEq, Eq)]
841pub struct WorkflowHistoryBudget {
842    pub event_count: u64,
843    pub size_bytes: Option<u64>,
844    pub continue_as_new_recommended: bool,
845    pub pressure: Option<String>,
846}
847
848#[doc(hidden)]
849#[derive(Clone, Debug)]
850pub struct ContinueAsNewRequest {
851    arguments: AvroValue,
852    options: ContinueAsNewOptions,
853}
854
855impl WorkflowCommandOptions {
856    pub fn new() -> Self {
857        Self::default()
858    }
859
860    pub fn reason(mut self, reason: impl Into<String>) -> Self {
861        self.reason = Some(reason.into());
862        self
863    }
864
865    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
866        self.request_id = Some(request_id.into());
867        self
868    }
869}
870
871/// The accepted, machine-readable result of a lifecycle command.
872#[derive(Clone, Debug, PartialEq)]
873pub struct WorkflowCommandResult {
874    pub command: WorkflowCommandKind,
875    pub workflow_id: String,
876    pub run_id: Option<String>,
877    pub outcome: Option<String>,
878    pub reason: Option<String>,
879    pub command_status: Option<String>,
880    pub raw: Value,
881}
882
883/// A stable rejection returned by instance- or selected-run lifecycle commands.
884#[derive(Clone, Debug, Error)]
885#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
886pub struct WorkflowCommandRejection {
887    pub command: WorkflowCommandKind,
888    pub status: u16,
889    pub reason: String,
890    pub message: String,
891    pub workflow_id: String,
892    pub run_id: Option<String>,
893    pub target_scope: Option<String>,
894    pub body: Value,
895}
896
897/// Stable terminal categories returned by [`WorkflowHandle::result`].
898#[derive(Clone, Copy, Debug, PartialEq, Eq)]
899pub enum WorkflowTerminalKind {
900    Failed,
901    Cancelled,
902    Terminated,
903    TimedOut,
904}
905
906/// A typed terminal workflow outcome with durable identity and failure metadata.
907///
908/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
909/// of parsing its display representation. Fields remain `None` when an older
910/// server did not publish that metadata.
911#[derive(Clone, Debug, Error)]
912#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
913pub struct WorkflowTerminalOutcome {
914    pub kind: WorkflowTerminalKind,
915    pub workflow_id: String,
916    pub run_id: Option<String>,
917    pub reason: String,
918    pub failure_category: Option<String>,
919    pub failure_id: Option<String>,
920    pub exception_type: Option<String>,
921    pub exception_class: Option<String>,
922    pub non_retryable: Option<bool>,
923    pub message: Option<String>,
924    pub exception: Option<Value>,
925    pub raw: Value,
926}
927
928/// A worker-side activity settlement or heartbeat rejected by durable state.
929#[derive(Clone, Debug, Error)]
930#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
931pub struct ActivityTaskRejection {
932    pub operation: String,
933    pub status: u16,
934    pub reason: String,
935    pub task_id: String,
936    pub activity_attempt_id: String,
937    pub cancel_requested: bool,
938    pub can_continue: Option<bool>,
939    pub run_closed_reason: Option<String>,
940    pub body: Value,
941}
942
943/// Stable validation categories for [`ActivityOptions`].
944#[derive(Clone, Copy, Debug, PartialEq, Eq)]
945pub enum ActivityOptionsErrorKind {
946    EmptyTaskQueue,
947    EmptyRetryPolicy,
948    InvalidMaxAttempts,
949    BackoffWithoutRetryBudget,
950    TooManyBackoffIntervals,
951    InvalidBackoffCoefficient,
952    BackoffGenerationTooLarge,
953    BackoffOverflow,
954    EmptyNonRetryableErrorType,
955    TimeoutNotPositive,
956    TimeoutOverflow,
957    TimeoutOrder,
958}
959
960/// A machine-readable activity-options validation failure.
961#[derive(Clone, Debug, Error, PartialEq, Eq)]
962#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
963pub struct ActivityOptionsError {
964    pub kind: ActivityOptionsErrorKind,
965    pub field: Option<&'static str>,
966    pub message: String,
967}
968
969impl ActivityOptionsError {
970    fn new(
971        kind: ActivityOptionsErrorKind,
972        field: Option<&'static str>,
973        message: impl Into<String>,
974    ) -> Self {
975        Self {
976            kind,
977            field,
978            message: message.into(),
979        }
980    }
981}
982
983/// Stable terminal categories returned when an awaited activity does not succeed.
984#[derive(Clone, Copy, Debug, PartialEq, Eq)]
985pub enum ActivityFailureKind {
986    Failed,
987    Cancelled,
988    TimedOut,
989}
990
991/// A stable, machine-readable terminal activity failure.
992///
993/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
994/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
995#[derive(Clone, Debug, Error)]
996#[error("activity failed ({reason}): {message}")]
997pub struct ActivityFailure {
998    pub kind: ActivityFailureKind,
999    pub reason: String,
1000    pub message: String,
1001    pub activity_execution_id: Option<String>,
1002    pub activity_attempt_id: Option<String>,
1003    pub activity_type: Option<String>,
1004    pub activity_class: Option<String>,
1005    pub attempt_number: Option<u64>,
1006    pub failure_id: Option<String>,
1007    pub failure_category: Option<String>,
1008    pub timeout_kind: Option<String>,
1009    pub non_retryable: bool,
1010    pub exception_type: Option<String>,
1011    pub exception_class: Option<String>,
1012    pub code: Option<Value>,
1013    pub exception: Option<Value>,
1014}
1015
1016/// Stable terminal categories returned when an awaited child does not succeed.
1017#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1018pub enum ChildWorkflowFailureKind {
1019    Failed,
1020    Cancelled,
1021    Terminated,
1022}
1023
1024/// A stable, machine-readable child workflow failure delivered to its parent.
1025///
1026/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
1027/// of parsing the display message. Child and parent identifiers retain the
1028/// relationship recorded in durable history across worker restarts.
1029#[derive(Clone, Debug, Error)]
1030#[error("child workflow failed ({reason}): {message}")]
1031pub struct ChildWorkflowFailure {
1032    pub kind: ChildWorkflowFailureKind,
1033    pub reason: String,
1034    pub message: String,
1035    pub parent_workflow_id: Option<String>,
1036    pub parent_workflow_run_id: Option<String>,
1037    pub child_workflow_id: Option<String>,
1038    pub child_workflow_run_id: Option<String>,
1039    pub child_workflow_type: Option<String>,
1040    pub failure_id: Option<String>,
1041    pub failure_category: Option<String>,
1042    pub exception_type: Option<String>,
1043    pub exception_class: Option<String>,
1044    pub non_retryable: bool,
1045    pub code: Option<Value>,
1046    pub exception: Option<Value>,
1047}
1048
1049/// The identity of one durable workflow execution.
1050#[derive(Clone, Debug, PartialEq, Eq)]
1051pub struct WorkflowIdentity {
1052    pub workflow_id: Option<String>,
1053    pub run_id: Option<String>,
1054}
1055
1056/// A successful child result together with its durable parent-child identity.
1057#[derive(Clone, Debug, PartialEq)]
1058pub struct ChildWorkflowResult {
1059    pub parent: WorkflowIdentity,
1060    pub child: WorkflowIdentity,
1061    pub child_workflow_type: Option<String>,
1062    pub result: Value,
1063}
1064
1065/// Lossless successful child result for fixed Avro Value workflows.
1066#[derive(Clone, Debug, PartialEq)]
1067pub struct ChildWorkflowAvroResult {
1068    pub parent: WorkflowIdentity,
1069    pub child: WorkflowIdentity,
1070    pub child_workflow_type: Option<String>,
1071    pub result: AvroValue,
1072}
1073
1074/// Stable user-facing identity for one member of a durable selection group.
1075#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1076#[serde(untagged)]
1077pub enum SelectionKey {
1078    Index(usize),
1079    Name(String),
1080}
1081
1082impl From<usize> for SelectionKey {
1083    fn from(value: usize) -> Self {
1084        Self::Index(value)
1085    }
1086}
1087
1088impl From<String> for SelectionKey {
1089    fn from(value: String) -> Self {
1090        Self::Name(value)
1091    }
1092}
1093
1094impl From<&str> for SelectionKey {
1095    fn from(value: &str) -> Self {
1096        Self::Name(value.to_string())
1097    }
1098}
1099
1100/// Typed result of explicitly awaiting a cancelled non-winning operation.
1101#[derive(Clone, Debug, Error, PartialEq, Eq)]
1102#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1103pub struct DurableOperationCancelled {
1104    pub selection_group_id: String,
1105    pub member_key: SelectionKey,
1106    pub member_index: usize,
1107    pub operation_kind: String,
1108    pub operation_identity: String,
1109}
1110
1111/// Stable identity for one enclosing deterministic parallel group.
1112///
1113/// The same fields are attached to every ordinary activity, timer, or child
1114/// workflow command in the group. Nested leaves carry an outer-to-inner path;
1115/// no Rust-specific wire command is introduced.
1116#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1117pub struct ParallelGroupMetadata {
1118    pub parallel_group_id: String,
1119    pub parallel_group_kind: String,
1120    pub parallel_group_base_sequence: u64,
1121    pub parallel_group_size: usize,
1122    pub parallel_group_index: usize,
1123    #[serde(default, skip_serializing_if = "Option::is_none")]
1124    pub parallel_group_mode: Option<String>,
1125    #[serde(default, skip_serializing_if = "Option::is_none")]
1126    pub selection_member_key: Option<SelectionKey>,
1127    #[serde(default, skip_serializing_if = "Option::is_none")]
1128    pub selection_member_index: Option<usize>,
1129    #[serde(default, skip_serializing_if = "Option::is_none")]
1130    pub selection_member_base_sequence: Option<u64>,
1131    #[serde(default, skip_serializing_if = "Option::is_none")]
1132    pub selection_member_size: Option<usize>,
1133    #[serde(default, skip_serializing_if = "Option::is_none")]
1134    pub selection_member_kind: Option<String>,
1135}
1136
1137/// One input-ordered result returned by [`WorkflowContext::parallel`].
1138#[derive(Clone, Debug, PartialEq)]
1139pub enum ParallelResult {
1140    Activity(Value),
1141    ChildWorkflow(ChildWorkflowResult),
1142    Timer,
1143    Signal(Vec<Value>),
1144    Condition(ConditionWaitResult),
1145    Group(Vec<ParallelResult>),
1146}
1147
1148/// Lossless fixed-Avro counterpart to [`ParallelResult`].
1149#[derive(Clone, Debug, PartialEq)]
1150pub enum ParallelAvroResult {
1151    Activity(AvroValue),
1152    ChildWorkflow(ChildWorkflowAvroResult),
1153    Timer,
1154    Signal(Vec<AvroValue>),
1155    Condition(ConditionWaitResult),
1156    Group(Vec<ParallelAvroResult>),
1157}
1158
1159impl ParallelAvroResult {
1160    fn into_json_result(self) -> Result<ParallelResult> {
1161        match self {
1162            Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1163            Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1164                parent: result.parent,
1165                child: result.child,
1166                child_workflow_type: result.child_workflow_type,
1167                result: result.result.into_json()?,
1168            })),
1169            Self::Timer => Ok(ParallelResult::Timer),
1170            Self::Signal(values) => Ok(ParallelResult::Signal(
1171                values
1172                    .into_iter()
1173                    .map(AvroValue::into_json)
1174                    .collect::<Result<Vec<_>>>()?,
1175            )),
1176            Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1177            Self::Group(results) => Ok(ParallelResult::Group(
1178                results
1179                    .into_iter()
1180                    .map(Self::into_json_result)
1181                    .collect::<Result<Vec<_>>>()?,
1182            )),
1183        }
1184    }
1185}
1186
1187/// One successful leaf retained when another parallel member failed.
1188#[derive(Clone, Debug, PartialEq)]
1189pub struct ParallelCompletion {
1190    pub member_path: Vec<usize>,
1191    pub result: ParallelResult,
1192}
1193
1194/// A deterministic join failed after some siblings had already completed.
1195///
1196/// `cause` retains the typed activity, child-workflow, cancellation, or codec
1197/// error. `completed` is declaration ordered and contains only durable
1198/// successes observed in the same replay. Late sibling completions can add
1199/// entries on a later replay without changing `member_path` or the selected
1200/// positional failure.
1201#[derive(Debug, Error)]
1202#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1203pub struct ParallelFailure {
1204    pub group_id: String,
1205    pub member_path: Vec<usize>,
1206    pub group_path: Vec<ParallelGroupMetadata>,
1207    pub completed: Vec<ParallelCompletion>,
1208    #[source]
1209    pub cause: Box<Error>,
1210}
1211
1212/// Stable validation error returned before an invalid group emits commands.
1213#[derive(Clone, Debug, Error, PartialEq, Eq)]
1214#[error("invalid deterministic parallel group ({reason}): {message}")]
1215pub struct ParallelGroupError {
1216    pub reason: &'static str,
1217    pub member_path: Vec<usize>,
1218    pub message: String,
1219}
1220
1221/// Cooperative workflow cancellation observed at an author-controlled point.
1222#[derive(Clone, Debug, Error, PartialEq, Eq)]
1223#[error("workflow cancellation was requested")]
1224pub struct WorkflowCancellationRequested;
1225
1226/// A forward saga failure followed by a terminal compensation failure.
1227#[derive(Debug, Error)]
1228#[error(
1229    "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1230)]
1231pub struct SagaCompensationFailure {
1232    pub initiating_failure: Box<Error>,
1233    pub compensation_failure: Box<Error>,
1234    pub compensation_activity_type: String,
1235    pub compensation_registration_order: usize,
1236}
1237
1238/// Server behavior when a parent closes while its child is still open.
1239#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1240pub enum ParentClosePolicy {
1241    #[default]
1242    Abandon,
1243    RequestCancel,
1244    Terminate,
1245}
1246
1247impl ParentClosePolicy {
1248    fn as_str(self) -> &'static str {
1249        match self {
1250            Self::Abandon => "abandon",
1251            Self::RequestCancel => "request_cancel",
1252            Self::Terminate => "terminate",
1253        }
1254    }
1255}
1256
1257/// Durable retry policy for one child workflow invocation.
1258#[derive(Clone, Debug, Default, PartialEq, Eq)]
1259pub struct ChildWorkflowRetryPolicy {
1260    pub max_attempts: Option<u32>,
1261    pub backoff_seconds: Vec<u64>,
1262    pub non_retryable_error_types: Vec<String>,
1263}
1264
1265/// Options recorded with a child-workflow command.
1266///
1267/// The task queue is mandatory so routing is explicit and replay-stable.
1268#[derive(Clone, Debug, PartialEq, Eq)]
1269pub struct ChildWorkflowOptions {
1270    pub task_queue: String,
1271    pub parent_close_policy: ParentClosePolicy,
1272    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1273    pub execution_timeout_seconds: Option<u64>,
1274    pub run_timeout_seconds: Option<u64>,
1275}
1276
1277impl ChildWorkflowOptions {
1278    pub fn new(task_queue: impl Into<String>) -> Self {
1279        Self {
1280            task_queue: task_queue.into(),
1281            parent_close_policy: ParentClosePolicy::Abandon,
1282            retry_policy: None,
1283            execution_timeout_seconds: None,
1284            run_timeout_seconds: None,
1285        }
1286    }
1287
1288    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1289        self.parent_close_policy = policy;
1290        self
1291    }
1292
1293    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1294        self.retry_policy = Some(policy);
1295        self
1296    }
1297
1298    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1299        self.execution_timeout_seconds = Some(seconds);
1300        self
1301    }
1302
1303    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1304        self.run_timeout_seconds = Some(seconds);
1305        self
1306    }
1307}
1308
1309/// Backoff intervals for one durable activity retry policy.
1310#[derive(Clone, Debug, PartialEq, Eq)]
1311pub enum ActivityBackoff {
1312    /// Use these intervals between attempts. The server repeats the final
1313    /// interval if the retry budget contains more attempts than entries.
1314    Explicit(Vec<Duration>),
1315    /// Generate one interval for every retry using integer exponential growth.
1316    Exponential {
1317        initial_interval: Duration,
1318        coefficient: u32,
1319        maximum_interval: Option<Duration>,
1320    },
1321}
1322
1323/// Durable server-side retry policy for one activity execution.
1324#[derive(Clone, Debug, Default, PartialEq, Eq)]
1325pub struct ActivityRetryPolicy {
1326    pub max_attempts: Option<u32>,
1327    pub backoff: Option<ActivityBackoff>,
1328    pub non_retryable_error_types: Vec<String>,
1329}
1330
1331impl ActivityRetryPolicy {
1332    /// Start a policy with a finite attempt budget, including the first attempt.
1333    pub fn new(max_attempts: u32) -> Self {
1334        Self {
1335            max_attempts: Some(max_attempts),
1336            ..Self::default()
1337        }
1338    }
1339
1340    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1341        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1342        self
1343    }
1344
1345    pub fn exponential_backoff(
1346        mut self,
1347        initial_interval: Duration,
1348        coefficient: u32,
1349        maximum_interval: Option<Duration>,
1350    ) -> Self {
1351        self.backoff = Some(ActivityBackoff::Exponential {
1352            initial_interval,
1353            coefficient,
1354            maximum_interval,
1355        });
1356        self
1357    }
1358
1359    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1360        self.non_retryable_error_types.push(error_type.into());
1361        self
1362    }
1363
1364    pub fn non_retryable_error_types(
1365        mut self,
1366        error_types: impl IntoIterator<Item = impl Into<String>>,
1367    ) -> Self {
1368        self.non_retryable_error_types
1369            .extend(error_types.into_iter().map(Into::into));
1370        self
1371    }
1372}
1373
1374/// Options recorded atomically on one deterministic `schedule_activity` command.
1375///
1376/// Durations are rounded up to whole seconds when encoded, so the server never
1377/// receives a shorter timeout or backoff than the caller requested.
1378#[derive(Clone, Debug, Default, PartialEq, Eq)]
1379pub struct ActivityOptions {
1380    pub task_queue: Option<String>,
1381    pub retry_policy: Option<ActivityRetryPolicy>,
1382    pub start_to_close_timeout: Option<Duration>,
1383    pub schedule_to_start_timeout: Option<Duration>,
1384    pub schedule_to_close_timeout: Option<Duration>,
1385    pub heartbeat_timeout: Option<Duration>,
1386}
1387
1388impl ActivityOptions {
1389    pub fn new() -> Self {
1390        Self::default()
1391    }
1392
1393    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1394        self.task_queue = Some(task_queue.into());
1395        self
1396    }
1397
1398    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1399        self.retry_policy = Some(policy);
1400        self
1401    }
1402
1403    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1404        self.start_to_close_timeout = Some(timeout);
1405        self
1406    }
1407
1408    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1409        self.schedule_to_start_timeout = Some(timeout);
1410        self
1411    }
1412
1413    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1414        self.schedule_to_close_timeout = Some(timeout);
1415        self
1416    }
1417
1418    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1419        self.heartbeat_timeout = Some(timeout);
1420        self
1421    }
1422
1423    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1424        if self
1425            .task_queue
1426            .as_deref()
1427            .is_some_and(|queue| queue.trim().is_empty())
1428        {
1429            return Err(ActivityOptionsError::new(
1430                ActivityOptionsErrorKind::EmptyTaskQueue,
1431                Some("task_queue"),
1432                "task_queue must not be empty",
1433            ));
1434        }
1435
1436        for (field, value) in [
1437            ("start_to_close_timeout", self.start_to_close_timeout),
1438            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1439            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1440            ("heartbeat_timeout", self.heartbeat_timeout),
1441        ] {
1442            if value.is_some_and(|value| value.is_zero()) {
1443                return Err(ActivityOptionsError::new(
1444                    ActivityOptionsErrorKind::TimeoutNotPositive,
1445                    Some(field),
1446                    format!("{field} must be positive"),
1447                ));
1448            }
1449        }
1450
1451        validate_timeout_order(
1452            "heartbeat_timeout",
1453            self.heartbeat_timeout,
1454            "start_to_close_timeout",
1455            self.start_to_close_timeout,
1456        )?;
1457        validate_timeout_order(
1458            "start_to_close_timeout",
1459            self.start_to_close_timeout,
1460            "schedule_to_close_timeout",
1461            self.schedule_to_close_timeout,
1462        )?;
1463        validate_timeout_order(
1464            "schedule_to_start_timeout",
1465            self.schedule_to_start_timeout,
1466            "schedule_to_close_timeout",
1467            self.schedule_to_close_timeout,
1468        )?;
1469
1470        Ok(ValidatedActivityOptions {
1471            task_queue: self.task_queue.clone(),
1472            retry_policy: self
1473                .retry_policy
1474                .as_ref()
1475                .map(validate_activity_retry_policy)
1476                .transpose()?,
1477            start_to_close_timeout: timeout_seconds(
1478                "start_to_close_timeout",
1479                self.start_to_close_timeout,
1480            )?,
1481            schedule_to_start_timeout: timeout_seconds(
1482                "schedule_to_start_timeout",
1483                self.schedule_to_start_timeout,
1484            )?,
1485            schedule_to_close_timeout: timeout_seconds(
1486                "schedule_to_close_timeout",
1487                self.schedule_to_close_timeout,
1488            )?,
1489            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1490        })
1491    }
1492}
1493
1494/// A deferred durable leaf or nested group for [`WorkflowContext::parallel`].
1495///
1496/// Constructors capture arguments but perform no I/O. The join validates the
1497/// complete tree, attaches the existing parallel-group metadata to every
1498/// ordinary command, schedules all leaves, and then suspends.
1499pub enum ParallelOperation {
1500    Activity {
1501        activity_type: String,
1502        options: ActivityOptions,
1503        arguments: Result<AvroValue>,
1504    },
1505    ChildWorkflow {
1506        workflow_type: String,
1507        options: ChildWorkflowOptions,
1508        arguments: Result<AvroValue>,
1509    },
1510    Timer(Duration),
1511    Signal(String),
1512    Condition {
1513        options: ConditionWaitOptions,
1514        predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1515    },
1516    Group(Vec<ParallelOperation>),
1517}
1518
1519impl ParallelOperation {
1520    pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1521        Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1522    }
1523
1524    pub fn activity_with_options<T: Serialize>(
1525        activity_type: impl Into<String>,
1526        options: ActivityOptions,
1527        args: T,
1528    ) -> Self {
1529        Self::Activity {
1530            activity_type: activity_type.into(),
1531            options,
1532            arguments: AvroValue::from_serialize(&args),
1533        }
1534    }
1535
1536    pub fn child_workflow<T: Serialize>(
1537        workflow_type: impl Into<String>,
1538        options: ChildWorkflowOptions,
1539        args: T,
1540    ) -> Self {
1541        Self::ChildWorkflow {
1542            workflow_type: workflow_type.into(),
1543            options,
1544            arguments: AvroValue::from_serialize(&args),
1545        }
1546    }
1547
1548    pub fn timer(duration: Duration) -> Self {
1549        Self::Timer(duration)
1550    }
1551
1552    pub fn signal(signal_name: impl Into<String>) -> Self {
1553        Self::Signal(signal_name.into())
1554    }
1555
1556    pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1557    where
1558        F: Fn() -> Result<bool> + Send + 'static,
1559    {
1560        Self::Condition {
1561            options,
1562            predicate: Box::new(predicate),
1563        }
1564    }
1565
1566    pub fn group(operations: Vec<ParallelOperation>) -> Self {
1567        Self::Group(operations)
1568    }
1569}
1570
1571#[derive(Clone, Debug)]
1572struct ValidatedActivityOptions {
1573    task_queue: Option<String>,
1574    retry_policy: Option<Value>,
1575    start_to_close_timeout: Option<u64>,
1576    schedule_to_start_timeout: Option<u64>,
1577    schedule_to_close_timeout: Option<u64>,
1578    heartbeat_timeout: Option<u64>,
1579}
1580
1581fn validate_timeout_order(
1582    smaller_name: &'static str,
1583    smaller: Option<Duration>,
1584    larger_name: &'static str,
1585    larger: Option<Duration>,
1586) -> std::result::Result<(), ActivityOptionsError> {
1587    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1588        return Err(ActivityOptionsError::new(
1589            ActivityOptionsErrorKind::TimeoutOrder,
1590            Some(smaller_name),
1591            format!("{smaller_name} must be <= {larger_name}"),
1592        ));
1593    }
1594    Ok(())
1595}
1596
1597fn timeout_seconds(
1598    field: &'static str,
1599    value: Option<Duration>,
1600) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1601    value
1602        .map(|value| {
1603            activity_protocol_seconds(value).ok_or_else(|| {
1604                ActivityOptionsError::new(
1605                    ActivityOptionsErrorKind::TimeoutOverflow,
1606                    Some(field),
1607                    format!("{field} is too large for the worker protocol"),
1608                )
1609            })
1610        })
1611        .transpose()
1612}
1613
1614fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1615    value
1616        .as_secs()
1617        .checked_add(u64::from(value.subsec_nanos() > 0))
1618}
1619
1620fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1621    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1622}
1623
1624fn validate_activity_retry_policy(
1625    policy: &ActivityRetryPolicy,
1626) -> std::result::Result<Value, ActivityOptionsError> {
1627    if policy.max_attempts.is_none()
1628        && policy.backoff.is_none()
1629        && policy.non_retryable_error_types.is_empty()
1630    {
1631        return Err(ActivityOptionsError::new(
1632            ActivityOptionsErrorKind::EmptyRetryPolicy,
1633            Some("retry_policy"),
1634            "retry_policy must configure at least one field",
1635        ));
1636    }
1637    if policy.max_attempts == Some(0) {
1638        return Err(ActivityOptionsError::new(
1639            ActivityOptionsErrorKind::InvalidMaxAttempts,
1640            Some("retry_policy.max_attempts"),
1641            "max_attempts must be >= 1",
1642        ));
1643    }
1644    if policy
1645        .non_retryable_error_types
1646        .iter()
1647        .any(|error_type| error_type.trim().is_empty())
1648    {
1649        return Err(ActivityOptionsError::new(
1650            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1651            Some("retry_policy.non_retryable_error_types"),
1652            "non_retryable_error_types must not contain empty values",
1653        ));
1654    }
1655
1656    let backoff_seconds = match &policy.backoff {
1657        None => None,
1658        Some(backoff) => {
1659            let max_attempts = policy.max_attempts.ok_or_else(|| {
1660                ActivityOptionsError::new(
1661                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1662                    Some("retry_policy.backoff"),
1663                    "backoff requires max_attempts",
1664                )
1665            })?;
1666            let retry_count = max_attempts.saturating_sub(1) as usize;
1667            let intervals = match backoff {
1668                ActivityBackoff::Explicit(intervals) => {
1669                    if intervals.len() > retry_count {
1670                        return Err(ActivityOptionsError::new(
1671                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
1672                            Some("retry_policy.backoff"),
1673                            "backoff interval count must not exceed max_attempts - 1",
1674                        ));
1675                    }
1676                    intervals.clone()
1677                }
1678                ActivityBackoff::Exponential {
1679                    initial_interval,
1680                    coefficient,
1681                    maximum_interval,
1682                } => {
1683                    if *coefficient < 1 {
1684                        return Err(ActivityOptionsError::new(
1685                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1686                            Some("retry_policy.backoff.coefficient"),
1687                            "backoff coefficient must be >= 1",
1688                        ));
1689                    }
1690                    if retry_count > 10_000 {
1691                        return Err(ActivityOptionsError::new(
1692                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1693                            Some("retry_policy.max_attempts"),
1694                            "generated backoff supports at most 10000 retry intervals",
1695                        ));
1696                    }
1697                    let mut current = *initial_interval;
1698                    let mut intervals = Vec::with_capacity(retry_count);
1699                    for _ in 0..retry_count {
1700                        let interval = maximum_interval
1701                            .map(|maximum| current.min(maximum))
1702                            .unwrap_or(current);
1703                        intervals.push(interval);
1704                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
1705                            break;
1706                        }
1707                        current = current.checked_mul(*coefficient).ok_or_else(|| {
1708                            ActivityOptionsError::new(
1709                                ActivityOptionsErrorKind::BackoffOverflow,
1710                                Some("retry_policy.backoff"),
1711                                "generated backoff interval overflowed",
1712                            )
1713                        })?;
1714                    }
1715                    intervals
1716                }
1717            };
1718            Some(
1719                intervals
1720                    .into_iter()
1721                    .map(|interval| {
1722                        activity_protocol_seconds(interval).ok_or_else(|| {
1723                            ActivityOptionsError::new(
1724                                ActivityOptionsErrorKind::BackoffOverflow,
1725                                Some("retry_policy.backoff"),
1726                                "backoff interval is too large for the worker protocol",
1727                            )
1728                        })
1729                    })
1730                    .collect::<std::result::Result<Vec<_>, _>>()?,
1731            )
1732        }
1733    };
1734
1735    let mut encoded = serde_json::Map::new();
1736    if let Some(max_attempts) = policy.max_attempts {
1737        encoded.insert("max_attempts".to_string(), json!(max_attempts));
1738    }
1739    if let Some(backoff_seconds) = backoff_seconds {
1740        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1741    }
1742    if !policy.non_retryable_error_types.is_empty() {
1743        let mut canonical_error_types = Vec::new();
1744        for error_type in policy
1745            .non_retryable_error_types
1746            .iter()
1747            .map(|error_type| error_type.trim())
1748        {
1749            if !canonical_error_types.contains(&error_type) {
1750                canonical_error_types.push(error_type);
1751            }
1752        }
1753        encoded.insert(
1754            "non_retryable_error_types".to_string(),
1755            json!(canonical_error_types),
1756        );
1757    }
1758    Ok(Value::Object(encoded))
1759}
1760
1761/// A stable, machine-readable failure raised when workflow code no longer
1762/// reconstructs the durable command stream recorded in history.
1763#[derive(Clone, Debug, Error)]
1764#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1765pub struct ReplayFailure {
1766    pub reason: String,
1767    pub sequence: Option<u64>,
1768    pub expected: Option<String>,
1769    pub actual: Option<String>,
1770    pub message: String,
1771}
1772
1773impl ReplayFailure {
1774    fn new(
1775        reason: impl Into<String>,
1776        sequence: Option<u64>,
1777        expected: Option<String>,
1778        actual: Option<String>,
1779        message: impl Into<String>,
1780    ) -> Self {
1781        Self {
1782            reason: reason.into(),
1783            sequence,
1784            expected,
1785            actual,
1786            message: message.into(),
1787        }
1788    }
1789}
1790
1791/// A stable, machine-readable workflow query or query-task settlement failure.
1792#[derive(Clone, Debug, Error)]
1793#[error("query failed ({reason}, HTTP {status}): {message}")]
1794pub struct QueryFailure {
1795    pub status: u16,
1796    pub reason: String,
1797    pub message: String,
1798    pub body: Value,
1799}
1800
1801/// A stable failure returned when a server rejects an SDK protocol version.
1802#[derive(Clone, Debug, Error)]
1803#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1804pub struct ProtocolFailure {
1805    pub status: u16,
1806    pub reason: String,
1807    pub message: String,
1808    pub supported_version: Option<String>,
1809    pub requested_version: Option<String>,
1810    pub body: Value,
1811}
1812
1813#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1814pub struct PayloadEnvelope {
1815    pub codec: String,
1816    pub blob: String,
1817}
1818
1819impl PayloadEnvelope {
1820    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1821        encode_payload(value, DEFAULT_CODEC)
1822    }
1823
1824    /// Encode an explicit typed value, including the bytes branch that JSON
1825    /// serialization cannot represent.
1826    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1827        encode_avro_value(value)
1828    }
1829}
1830
1831/// Native adapter for the fixed language-neutral Avro Value schema.
1832#[derive(Clone, Debug)]
1833pub enum AvroValue {
1834    Null,
1835    Boolean(bool),
1836    Long(i64),
1837    Double(f64),
1838    Bytes(Vec<u8>),
1839    String(String),
1840    Array(Vec<AvroValue>),
1841    Map(BTreeMap<String, AvroValue>),
1842}
1843
1844impl PartialEq for AvroValue {
1845    fn eq(&self, other: &Self) -> bool {
1846        match (self, other) {
1847            (Self::Null, Self::Null) => true,
1848            (Self::Boolean(left), Self::Boolean(right)) => left == right,
1849            (Self::Long(left), Self::Long(right)) => left == right,
1850            (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1851            (Self::Bytes(left), Self::Bytes(right)) => left == right,
1852            (Self::String(left), Self::String(right)) => left == right,
1853            (Self::Array(left), Self::Array(right)) => left == right,
1854            (Self::Map(left), Self::Map(right)) => left == right,
1855            _ => false,
1856        }
1857    }
1858}
1859
1860impl AvroValue {
1861    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1862        Self::from_serde_value(
1863            serde_value::to_value(value).map_err(|error| {
1864                Error::Codec(format!("could not adapt value for Avro: {error}"))
1865            })?,
1866        )
1867    }
1868
1869    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1870        use serde_value::Value as SerdeValue;
1871
1872        match value {
1873            SerdeValue::Unit => Ok(Self::Null),
1874            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1875            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1876            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1877            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1878            SerdeValue::I64(value) => Ok(Self::Long(value)),
1879            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1880            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1881            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1882            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1883                Error::Codec(
1884                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1885                        .to_string(),
1886                )
1887            }),
1888            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1889            SerdeValue::F64(value) => Self::finite_double(value),
1890            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1891            SerdeValue::String(value) => Ok(Self::String(value)),
1892            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1893            SerdeValue::Option(None) => Ok(Self::Null),
1894            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1895                Self::from_serde_value(*value)
1896            }
1897            SerdeValue::Seq(values) => values
1898                .into_iter()
1899                .map(Self::from_serde_value)
1900                .collect::<Result<Vec<_>>>()
1901                .map(Self::Array),
1902            SerdeValue::Map(values) => values
1903                .into_iter()
1904                .map(|(key, value)| {
1905                    let SerdeValue::String(key) = key else {
1906                        return Err(Error::Codec(
1907                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1908                        ));
1909                    };
1910
1911                    Ok((key, Self::from_serde_value(value)?))
1912                })
1913                .collect::<Result<BTreeMap<_, _>>>()
1914                .map(Self::Map),
1915        }
1916    }
1917
1918    fn finite_double(value: f64) -> Result<Self> {
1919        if !value.is_finite() {
1920            return Err(Error::Codec(
1921                "non_finite_float: Avro Value doubles must be finite".to_string(),
1922            ));
1923        }
1924
1925        Ok(Self::Double(value))
1926    }
1927
1928    fn into_json(self) -> Result<Value> {
1929        match self {
1930            Self::Null => Ok(Value::Null),
1931            Self::Boolean(value) => Ok(Value::Bool(value)),
1932            Self::Long(value) => Ok(Value::Number(value.into())),
1933            Self::Double(value) => serde_json::Number::from_f64(value)
1934                .map(Value::Number)
1935                .ok_or_else(|| {
1936                    Error::Codec(
1937                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1938                    )
1939                }),
1940            Self::Bytes(value) => Ok(json!({
1941                "$type": "bytes",
1942                "base64": BASE64.encode(value),
1943            })),
1944            Self::String(value) => Ok(Value::String(value)),
1945            Self::Array(values) => values
1946                .into_iter()
1947                .map(Self::into_json)
1948                .collect::<Result<Vec<_>>>()
1949                .map(Value::Array),
1950            Self::Map(values) => values
1951                .into_iter()
1952                .map(|(key, value)| Ok((key, value.into_json()?)))
1953                .collect::<Result<serde_json::Map<_, _>>>()
1954                .map(Value::Object),
1955        }
1956    }
1957
1958    fn into_serde_value(self) -> serde_value::Value {
1959        use serde_value::Value as SerdeValue;
1960
1961        match self {
1962            Self::Null => SerdeValue::Unit,
1963            Self::Boolean(value) => SerdeValue::Bool(value),
1964            Self::Long(value) => SerdeValue::I64(value),
1965            Self::Double(value) => SerdeValue::F64(value),
1966            Self::Bytes(value) => SerdeValue::Bytes(value),
1967            Self::String(value) => SerdeValue::String(value),
1968            Self::Array(values) => {
1969                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1970            }
1971            Self::Map(values) => SerdeValue::Map(
1972                values
1973                    .into_iter()
1974                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1975                    .collect(),
1976            ),
1977        }
1978    }
1979
1980    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1981        self.into_serde_value().deserialize_into().map_err(|error| {
1982            Error::Codec(format!(
1983                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1984            ))
1985        })
1986    }
1987}
1988
1989impl Serialize for AvroValue {
1990    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1991    where
1992        S: Serializer,
1993    {
1994        match self {
1995            Self::Null => serializer.serialize_unit(),
1996            Self::Boolean(value) => serializer.serialize_bool(*value),
1997            Self::Long(value) => serializer.serialize_i64(*value),
1998            Self::Double(value) => serializer.serialize_f64(*value),
1999            Self::Bytes(value) => serializer.serialize_bytes(value),
2000            Self::String(value) => serializer.serialize_str(value),
2001            Self::Array(values) => {
2002                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2003                for value in values {
2004                    sequence.serialize_element(value)?;
2005                }
2006                sequence.end()
2007            }
2008            Self::Map(values) => {
2009                let mut map = serializer.serialize_map(Some(values.len()))?;
2010                for (key, value) in values {
2011                    map.serialize_entry(key, value)?;
2012                }
2013                map.end()
2014            }
2015        }
2016    }
2017}
2018
2019pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2020    let datum = avro_value_to_datum(value)?;
2021    let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2022        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2023    let mut bytes = Vec::with_capacity(datum.len() + 10);
2024    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2025    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2026    bytes.extend_from_slice(&datum);
2027    Ok(PayloadEnvelope {
2028        codec: DEFAULT_CODEC.to_string(),
2029        blob: BASE64.encode(bytes),
2030    })
2031}
2032
2033pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2034    if envelope.codec != DEFAULT_CODEC {
2035        return Err(unsupported_payload_codec(&envelope.codec));
2036    }
2037    decode_avro_value_blob(&envelope.blob)
2038}
2039
2040pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2041    let blob = match codec {
2042        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2043        other => return Err(unsupported_payload_codec(other)),
2044    };
2045
2046    Ok(PayloadEnvelope {
2047        codec: codec.to_string(),
2048        blob,
2049    })
2050}
2051
2052pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2053    match envelope.codec.as_str() {
2054        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2055        other => Err(unsupported_payload_codec(other)),
2056    }
2057}
2058
2059fn handler_type_error<T>(
2060    handler_kind: HandlerKind,
2061    handler_name: &str,
2062    value_kind: HandlerValueKind,
2063    message: impl Into<String>,
2064) -> Error {
2065    Error::HandlerType {
2066        handler_kind,
2067        handler_name: handler_name.to_string(),
2068        value_kind,
2069        rust_type: type_name::<T>(),
2070        message: message.into(),
2071    }
2072}
2073
2074fn decode_handler_input<T: DeserializeOwned>(
2075    arguments: AvroValue,
2076    handler_kind: HandlerKind,
2077    handler_name: &str,
2078) -> Result<T> {
2079    let argument = match arguments {
2080        AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2081            arguments.pop().expect("one typed handler argument")
2082        }
2083        AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2084        AvroValue::Array(arguments) => {
2085            return Err(handler_type_error::<T>(
2086                handler_kind,
2087                handler_name,
2088                HandlerValueKind::Input,
2089                format!(
2090                    "typed handlers accept one request value, but the task carried {} arguments",
2091                    arguments.len()
2092                ),
2093            ));
2094        }
2095        argument => argument,
2096    };
2097
2098    argument.deserialize().map_err(|error| {
2099        handler_type_error::<T>(
2100            handler_kind,
2101            handler_name,
2102            HandlerValueKind::Input,
2103            error.to_string(),
2104        )
2105    })
2106}
2107
2108fn encode_handler_result<T: Serialize>(
2109    result: &T,
2110    handler_kind: HandlerKind,
2111    handler_name: &str,
2112) -> Result<AvroValue> {
2113    AvroValue::from_serialize(result).map_err(|error| {
2114        handler_type_error::<T>(
2115            handler_kind,
2116            handler_name,
2117            HandlerValueKind::Result,
2118            error.to_string(),
2119        )
2120    })
2121}
2122
2123fn decode_handler_result<T: DeserializeOwned>(
2124    result: AvroValue,
2125    handler_kind: HandlerKind,
2126    handler_name: &str,
2127) -> Result<T> {
2128    result.deserialize().map_err(|error| {
2129        handler_type_error::<T>(
2130            handler_kind,
2131            handler_name,
2132            HandlerValueKind::Result,
2133            error.to_string(),
2134        )
2135    })
2136}
2137
2138#[cfg(test)]
2139fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2140    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2141}
2142
2143fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2144    validate_payload_codec(fallback_codec)?;
2145
2146    if value.is_null() {
2147        return Ok(Value::Null);
2148    }
2149
2150    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2151        return decode_blob(blob, codec);
2152    }
2153
2154    if let Some(blob) = value.as_str() {
2155        return decode_blob(blob, fallback_codec);
2156    }
2157
2158    Err(untagged_payload_value())
2159}
2160
2161fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2162    let envelope = match codec {
2163        DEFAULT_CODEC => encode_avro_value(value)?,
2164        other => return Err(unsupported_payload_codec(other)),
2165    };
2166    Ok(serde_json::to_value(envelope)?)
2167}
2168
2169fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2170    validate_payload_codec(fallback_codec)?;
2171
2172    if value.is_null() {
2173        return Ok(AvroValue::Null);
2174    }
2175
2176    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2177        validate_payload_codec(codec)?;
2178        return decode_avro_value_blob(blob);
2179    }
2180
2181    if let Some(blob) = value.as_str() {
2182        return match fallback_codec {
2183            DEFAULT_CODEC => decode_avro_value_blob(blob),
2184            other => Err(unsupported_payload_codec(other)),
2185        };
2186    }
2187
2188    Err(untagged_payload_value())
2189}
2190
2191fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2192    match value {
2193        AvroValue::Null => AvroValue::Array(Vec::new()),
2194        AvroValue::Array(_) => value,
2195        other => AvroValue::Array(vec![other]),
2196    }
2197}
2198
2199fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2200    match codec {
2201        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2202        other => Err(unsupported_payload_codec(other)),
2203    }
2204}
2205
2206fn validate_payload_codec(codec: &str) -> Result<()> {
2207    match codec {
2208        DEFAULT_CODEC => Ok(()),
2209        MISSING_TASK_PAYLOAD_CODEC => {
2210            Err(invalid_task_payload_codec("task payload_codec is missing"))
2211        }
2212        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2213        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2214            "task payload_codec must be a string",
2215        )),
2216        other => Err(unsupported_payload_codec(other)),
2217    }
2218}
2219
2220fn invalid_task_payload_codec(reason: &str) -> Error {
2221    Error::Codec(format!(
2222        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2223    ))
2224}
2225
2226fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2227    let Some(object) = value.as_object() else {
2228        return Ok(None);
2229    };
2230    if !object.contains_key("codec") && !object.contains_key("blob") {
2231        return Ok(None);
2232    }
2233
2234    let codec = object
2235        .get("codec")
2236        .and_then(Value::as_str)
2237        .ok_or_else(invalid_payload_envelope)?;
2238    validate_payload_codec(codec)?;
2239    let blob = object
2240        .get("blob")
2241        .and_then(Value::as_str)
2242        .ok_or_else(invalid_payload_envelope)?;
2243    Ok(Some((codec, blob)))
2244}
2245
2246fn invalid_payload_envelope() -> Error {
2247    Error::Codec(
2248        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2249            .to_string(),
2250    )
2251}
2252
2253fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2254    for command in commands {
2255        let Some(command) = command.as_object() else {
2256            continue;
2257        };
2258        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2259            continue;
2260        };
2261        let Some(payload_field) = workflow_command_payload_field(command_type) else {
2262            continue;
2263        };
2264
2265        if let Some(codec) = command.get("payload_codec") {
2266            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2267            validate_payload_codec(codec)?;
2268        }
2269
2270        let payload = command
2271            .get(payload_field)
2272            .ok_or_else(invalid_payload_envelope)?;
2273        validate_outbound_payload_envelope(payload)?;
2274    }
2275    Ok(())
2276}
2277
2278fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2279    if commands.iter().any(|command| {
2280        command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2281            && command
2282                .get("condition_wait_occurrence_id")
2283                .and_then(Value::as_str)
2284                .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2285    }) {
2286        CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2287    } else if commands.iter().any(|command| {
2288        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2289            && command.get("attribute_types").is_some()
2290    }) {
2291        TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2292    } else if commands
2293        .iter()
2294        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2295    {
2296        MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2297    } else if commands
2298        .iter()
2299        .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2300    {
2301        CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2302    } else if commands.iter().any(|command| {
2303        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2304    }) {
2305        SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2306    } else {
2307        WORKER_PROTOCOL_VERSION
2308    }
2309}
2310
2311fn workflow_completion_protocol_version_with_message_streams(
2312    commands: &[Value],
2313    has_message_stream_metadata: bool,
2314) -> &'static str {
2315    let command_protocol = workflow_completion_protocol_version(commands);
2316    if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2317        MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2318    } else {
2319        command_protocol
2320    }
2321}
2322
2323fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2324    match command_type {
2325        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2326        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2327        "start_service_operation" => Some("request_payload"),
2328        "upsert_memo" => Some("entries"),
2329        _ => None,
2330    }
2331}
2332
2333fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2334    let Some((codec, blob)) = payload_envelope_parts(value)? else {
2335        return Err(untagged_payload_value());
2336    };
2337    validate_payload_codec(codec)?;
2338    decode_avro_value_blob(blob)?;
2339    Ok(())
2340}
2341
2342fn unsupported_payload_codec(codec: &str) -> Error {
2343    Error::Codec(format!(
2344        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2345    ))
2346}
2347
2348fn untagged_payload_value() -> Error {
2349    Error::Codec(
2350        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2351            .to_string(),
2352    )
2353}
2354
2355fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2356    let bytes = BASE64.decode(blob).map_err(|err| {
2357        Error::Codec(format!(
2358            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2359        ))
2360    })?;
2361
2362    if serde_json::from_slice::<Value>(&bytes).is_ok() {
2363        return Err(unsupported_payload_codec("json"));
2364    }
2365
2366    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2367        return Err(Error::Codec(
2368            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2369        ));
2370    }
2371
2372    let fingerprint: [u8; 8] = bytes[2..10]
2373        .try_into()
2374        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2375    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2376        return Err(Error::Codec(format!(
2377            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2378            fingerprint
2379                .iter()
2380                .map(|byte| format!("{byte:02x}"))
2381                .collect::<String>()
2382        )));
2383    }
2384
2385    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2386    // The current fingerprint selects the current immutable schema, so reader
2387    // resolution would only re-walk the same recursive union. Future retained
2388    // writer fingerprints supply a distinct reader schema in this branch.
2389    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2390    if datum_reader.truncated {
2391        return Err(Error::Codec(
2392            "invalid_payload_framing: truncated Avro Value datum".to_string(),
2393        ));
2394    }
2395    let datum = datum.map_err(|err| {
2396        Error::Codec(format!(
2397            "invalid_payload_framing: malformed Avro Value datum: {err}"
2398        ))
2399    })?;
2400    if datum_reader.remaining() != 0 {
2401        return Err(Error::Codec(format!(
2402            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2403            datum_reader.remaining()
2404        )));
2405    }
2406    avro_value_from_datum(datum)
2407}
2408
2409struct StrictAvroDatumReader<'a> {
2410    bytes: &'a [u8],
2411    offset: usize,
2412    truncated: bool,
2413}
2414
2415impl<'a> StrictAvroDatumReader<'a> {
2416    fn new(bytes: &'a [u8]) -> Self {
2417        Self {
2418            bytes,
2419            offset: 0,
2420            truncated: false,
2421        }
2422    }
2423
2424    fn remaining(&self) -> usize {
2425        self.bytes.len() - self.offset
2426    }
2427}
2428
2429impl Read for StrictAvroDatumReader<'_> {
2430    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2431        let count = buffer.len().min(self.remaining());
2432        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2433        self.offset += count;
2434        if count < buffer.len() {
2435            self.truncated = true;
2436        }
2437
2438        Ok(count)
2439    }
2440}
2441
2442fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2443    let branch = match value {
2444        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2445        AvroValue::Boolean(value) => AvroDatum::Union(
2446            1,
2447            Box::new(AvroDatum::Record(vec![(
2448                "boolean".to_string(),
2449                AvroDatum::Boolean(*value),
2450            )])),
2451        ),
2452        AvroValue::Long(value) => AvroDatum::Union(
2453            2,
2454            Box::new(AvroDatum::Record(vec![(
2455                "long".to_string(),
2456                AvroDatum::Long(*value),
2457            )])),
2458        ),
2459        AvroValue::Double(value) => {
2460            if !value.is_finite() {
2461                return Err(Error::Codec(
2462                    "non_finite_float: Avro Value doubles must be finite".to_string(),
2463                ));
2464            }
2465            AvroDatum::Union(
2466                3,
2467                Box::new(AvroDatum::Record(vec![(
2468                    "double".to_string(),
2469                    AvroDatum::Double(*value),
2470                )])),
2471            )
2472        }
2473        AvroValue::Bytes(value) => AvroDatum::Union(
2474            4,
2475            Box::new(AvroDatum::Record(vec![(
2476                "bytes".to_string(),
2477                AvroDatum::Bytes(value.clone()),
2478            )])),
2479        ),
2480        AvroValue::String(value) => AvroDatum::Union(
2481            5,
2482            Box::new(AvroDatum::Record(vec![(
2483                "string".to_string(),
2484                AvroDatum::String(value.clone()),
2485            )])),
2486        ),
2487        AvroValue::Array(values) => AvroDatum::Union(
2488            6,
2489            Box::new(AvroDatum::Record(vec![(
2490                "items".to_string(),
2491                AvroDatum::Array(
2492                    values
2493                        .iter()
2494                        .map(avro_value_to_datum)
2495                        .collect::<Result<Vec<_>>>()?,
2496                ),
2497            )])),
2498        ),
2499        AvroValue::Map(values) => AvroDatum::Union(
2500            7,
2501            Box::new(AvroDatum::Record(vec![(
2502                "entries".to_string(),
2503                AvroDatum::Array(
2504                    values
2505                        .iter()
2506                        .map(|(key, value)| {
2507                            Ok(AvroDatum::Record(vec![
2508                                ("key".to_string(), AvroDatum::String(key.clone())),
2509                                ("value".to_string(), avro_value_to_datum(value)?),
2510                            ]))
2511                        })
2512                        .collect::<Result<Vec<_>>>()?,
2513                ),
2514            )])),
2515        ),
2516    };
2517    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2518}
2519
2520fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2521    let AvroDatum::Record(mut outer) = datum else {
2522        return Err(Error::Codec(
2523            "invalid_payload_framing: datum is not a Value record".to_string(),
2524        ));
2525    };
2526    let (_, branch) = outer
2527        .pop()
2528        .filter(|(name, _)| name == "value")
2529        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2530    let AvroDatum::Union(_, branch) = branch else {
2531        return Err(Error::Codec(
2532            "invalid_payload_framing: invalid Value union".to_string(),
2533        ));
2534    };
2535    match *branch {
2536        AvroDatum::Null => Ok(AvroValue::Null),
2537        AvroDatum::Record(mut fields) => {
2538            let (name, value) = fields.pop().ok_or_else(|| {
2539                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2540            })?;
2541            match (name.as_str(), value) {
2542                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2543                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2544                ("double", AvroDatum::Double(value)) if value.is_finite() => {
2545                    Ok(AvroValue::Double(value))
2546                }
2547                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2548                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2549                ("items", AvroDatum::Array(values)) => values
2550                    .into_iter()
2551                    .map(avro_value_from_datum)
2552                    .collect::<Result<Vec<_>>>()
2553                    .map(AvroValue::Array),
2554                ("entries", AvroDatum::Map(values)) => values
2555                    .into_iter()
2556                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2557                    .collect::<Result<BTreeMap<_, _>>>()
2558                    .map(AvroValue::Map),
2559                _ => Err(Error::Codec(
2560                    "invalid_payload_framing: unknown Value branch".to_string(),
2561                )),
2562            }
2563        }
2564        _ => Err(Error::Codec(
2565            "invalid_payload_framing: invalid Value branch".to_string(),
2566        )),
2567    }
2568}
2569
2570fn avro_value_schema() -> Result<&'static Schema> {
2571    match AVRO_VALUE_SCHEMA.get_or_init(|| {
2572        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2573            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2574    }) {
2575        Ok(schema) => Ok(schema),
2576        Err(message) => Err(Error::Codec(message.clone())),
2577    }
2578}
2579
2580fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2581    match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2582        // Apache Avro's Value::Map uses a randomized HashMap. Arrays and maps
2583        // have the same block representation when each ordered array record
2584        // contains the map key followed by its value, so this encoding-only
2585        // adaptation lets the official encoder retain BTreeMap wire order.
2586        let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2587            .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2588        let entries_schema = schema
2589            .pointer_mut("/fields/0/type/7/fields/0/type")
2590            .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2591        if *entries_schema != json!({"type": "map", "values": "Value"}) {
2592            return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2593        }
2594        *entries_schema = json!({
2595            "type": "array",
2596            "items": {
2597                "type": "record",
2598                "name": "MapEntry",
2599                "fields": [
2600                    {"name": "key", "type": "string"},
2601                    {"name": "value", "type": "Value"}
2602                ]
2603            }
2604        });
2605        Schema::parse_str(&schema.to_string())
2606            .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2607    }) {
2608        Ok(schema) => Ok(schema),
2609        Err(message) => Err(Error::Codec(message.clone())),
2610    }
2611}
2612
2613#[derive(Clone, Debug)]
2614pub struct Client {
2615    http: reqwest::Client,
2616    base_url: String,
2617    token: Option<String>,
2618    control_token: Option<String>,
2619    worker_token: Option<String>,
2620    namespace: String,
2621}
2622
2623impl Client {
2624    pub fn new(base_url: impl Into<String>) -> Result<Self> {
2625        Self::builder(base_url).build()
2626    }
2627
2628    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2629        ClientBuilder {
2630            base_url: base_url.into(),
2631            token: None,
2632            control_token: None,
2633            worker_token: None,
2634            namespace: "default".to_string(),
2635            timeout: Duration::from_secs(60),
2636        }
2637    }
2638
2639    pub async fn health(&self) -> Result<Value> {
2640        self.request_json(
2641            reqwest::Method::GET,
2642            "/health",
2643            RequestProtocol::ControlPlane,
2644            Option::<&Value>::None,
2645        )
2646        .await
2647    }
2648
2649    pub async fn cluster_info(&self) -> Result<Value> {
2650        self.request_json(
2651            reqwest::Method::GET,
2652            "/cluster/info",
2653            RequestProtocol::ControlPlane,
2654            Option::<&Value>::None,
2655        )
2656        .await
2657    }
2658
2659    pub async fn start_workflow<T: Serialize>(
2660        &self,
2661        workflow_type: &str,
2662        task_queue: &str,
2663        workflow_id: &str,
2664        input: T,
2665    ) -> Result<WorkflowHandle> {
2666        self.start_workflow_with_options(
2667            workflow_type,
2668            task_queue,
2669            workflow_id,
2670            WorkflowStartOptions::default(),
2671            input,
2672        )
2673        .await
2674    }
2675
2676    /// Start a workflow with explicit server-enforced execution and run
2677    /// deadlines.
2678    pub async fn start_workflow_with_options<T: Serialize>(
2679        &self,
2680        workflow_type: &str,
2681        task_queue: &str,
2682        workflow_id: &str,
2683        options: WorkflowStartOptions,
2684        input: T,
2685    ) -> Result<WorkflowHandle> {
2686        options.validate()?;
2687        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2688        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2689        let body = json!({
2690            "workflow_id": workflow_id,
2691            "workflow_type": workflow_type,
2692            "task_queue": task_queue,
2693            "input": input_envelope,
2694            "execution_timeout_seconds": options.execution_timeout_seconds,
2695            "run_timeout_seconds": options.run_timeout_seconds
2696        });
2697
2698        let data: Value = self
2699            .request_json(
2700                reqwest::Method::POST,
2701                "/workflows",
2702                RequestProtocol::ControlPlane,
2703                Some(&body),
2704            )
2705            .await?;
2706
2707        Ok(WorkflowHandle {
2708            client: self.clone(),
2709            workflow_id: data
2710                .get("workflow_id")
2711                .and_then(Value::as_str)
2712                .unwrap_or(workflow_id)
2713                .to_string(),
2714            run_id: data
2715                .get("run_id")
2716                .and_then(Value::as_str)
2717                .map(str::to_string),
2718            workflow_type: data
2719                .get("workflow_type")
2720                .and_then(Value::as_str)
2721                .unwrap_or(workflow_type)
2722                .to_string(),
2723        })
2724    }
2725
2726    pub async fn signal_workflow<T: Serialize>(
2727        &self,
2728        workflow_id: &str,
2729        signal_name: &str,
2730        input: T,
2731    ) -> Result<Value> {
2732        self.signal_workflow_target(workflow_id, None, signal_name, input)
2733            .await
2734    }
2735
2736    /// Append one idempotently identified item to an instance-scoped input stream.
2737    pub async fn append_message_stream<T: Serialize>(
2738        &self,
2739        workflow_id: &str,
2740        stream_name: &str,
2741        message_id: &str,
2742        input: T,
2743    ) -> Result<Value> {
2744        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2745        let body = json!({
2746            "message_id": message_id,
2747            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2748        });
2749        self.request_json(
2750            reqwest::Method::POST,
2751            &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2752            RequestProtocol::ControlPlane,
2753            Some(&body),
2754        )
2755        .await
2756    }
2757
2758    /// Signal only if `run_id` is still the current run for this instance.
2759    pub async fn signal_workflow_run<T: Serialize>(
2760        &self,
2761        workflow_id: &str,
2762        run_id: &str,
2763        signal_name: &str,
2764        input: T,
2765    ) -> Result<Value> {
2766        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2767            .await
2768    }
2769
2770    async fn signal_workflow_target<T: Serialize>(
2771        &self,
2772        workflow_id: &str,
2773        run_id: Option<&str>,
2774        signal_name: &str,
2775        input: T,
2776    ) -> Result<Value> {
2777        validate_user_signal_name(signal_name)?;
2778        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2779        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2780        let body = json!({
2781            "input": input_envelope
2782        });
2783        let path = match run_id {
2784            Some(run_id) => {
2785                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2786            }
2787            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2788        };
2789        self.request_json(
2790            reqwest::Method::POST,
2791            &path,
2792            RequestProtocol::ControlPlane,
2793            Some(&body),
2794        )
2795        .await
2796    }
2797
2798    /// Request cooperative cancellation of the current run for an instance.
2799    pub async fn cancel_workflow(
2800        &self,
2801        workflow_id: &str,
2802        options: WorkflowCommandOptions,
2803    ) -> Result<WorkflowCommandResult> {
2804        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2805            .await
2806    }
2807
2808    /// Request cooperative cancellation only if `run_id` is still current.
2809    pub async fn cancel_workflow_run(
2810        &self,
2811        workflow_id: &str,
2812        run_id: &str,
2813        options: WorkflowCommandOptions,
2814    ) -> Result<WorkflowCommandResult> {
2815        self.workflow_command(
2816            workflow_id,
2817            Some(run_id),
2818            WorkflowCommandKind::Cancel,
2819            options,
2820        )
2821        .await
2822    }
2823
2824    /// Forcefully terminate the current run for an instance.
2825    pub async fn terminate_workflow(
2826        &self,
2827        workflow_id: &str,
2828        options: WorkflowCommandOptions,
2829    ) -> Result<WorkflowCommandResult> {
2830        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2831            .await
2832    }
2833
2834    /// Forcefully terminate only if `run_id` is still current.
2835    pub async fn terminate_workflow_run(
2836        &self,
2837        workflow_id: &str,
2838        run_id: &str,
2839        options: WorkflowCommandOptions,
2840    ) -> Result<WorkflowCommandResult> {
2841        self.workflow_command(
2842            workflow_id,
2843            Some(run_id),
2844            WorkflowCommandKind::Terminate,
2845            options,
2846        )
2847        .await
2848    }
2849
2850    async fn workflow_command(
2851        &self,
2852        workflow_id: &str,
2853        run_id: Option<&str>,
2854        command: WorkflowCommandKind,
2855        options: WorkflowCommandOptions,
2856    ) -> Result<WorkflowCommandResult> {
2857        let path = match run_id {
2858            Some(run_id) => format!(
2859                "/workflows/{workflow_id}/runs/{run_id}/{}",
2860                command.as_str()
2861            ),
2862            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2863        };
2864        let data = match self
2865            .request_json(
2866                reqwest::Method::POST,
2867                &path,
2868                RequestProtocol::ControlPlane,
2869                Some(&options),
2870            )
2871            .await
2872        {
2873            Ok(data) => data,
2874            Err(Error::Http { status, body }) => {
2875                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2876                    command,
2877                    status,
2878                    body,
2879                    workflow_id,
2880                    run_id,
2881                )));
2882            }
2883            Err(error) => return Err(error),
2884        };
2885
2886        Ok(workflow_command_result(command, data, workflow_id, run_id))
2887    }
2888
2889    /// Execute a named, read-only query against a running or completed workflow.
2890    ///
2891    /// Arguments and results use the platform payload envelope. Server and
2892    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
2893    /// reason, HTTP status, and original response body.
2894    pub async fn query_workflow<T: Serialize>(
2895        &self,
2896        workflow_id: &str,
2897        query_name: &str,
2898        input: T,
2899    ) -> Result<Value> {
2900        self.query_workflow_target(workflow_id, None, query_name, input)
2901            .await
2902    }
2903
2904    /// Query only if `run_id` is still current, preventing accidental retargeting.
2905    pub async fn query_workflow_run<T: Serialize>(
2906        &self,
2907        workflow_id: &str,
2908        run_id: &str,
2909        query_name: &str,
2910        input: T,
2911    ) -> Result<Value> {
2912        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2913            .await
2914    }
2915
2916    /// Query a workflow and return the lossless fixed Avro Value result.
2917    pub async fn query_workflow_avro_value<T: Serialize>(
2918        &self,
2919        workflow_id: &str,
2920        query_name: &str,
2921        input: T,
2922    ) -> Result<AvroValue> {
2923        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2924            .await
2925    }
2926
2927    /// Query a selected run and return the lossless fixed Avro Value result.
2928    pub async fn query_workflow_run_avro_value<T: Serialize>(
2929        &self,
2930        workflow_id: &str,
2931        run_id: &str,
2932        query_name: &str,
2933        input: T,
2934    ) -> Result<AvroValue> {
2935        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2936            .await
2937    }
2938
2939    async fn query_workflow_avro_value_target<T: Serialize>(
2940        &self,
2941        workflow_id: &str,
2942        run_id: Option<&str>,
2943        query_name: &str,
2944        input: T,
2945    ) -> Result<AvroValue> {
2946        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2947        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2948        let path = match run_id {
2949            Some(run_id) => {
2950                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2951            }
2952            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2953        };
2954        let response: Value = match self
2955            .request_json(
2956                reqwest::Method::POST,
2957                &path,
2958                RequestProtocol::ControlPlane,
2959                Some(&body),
2960            )
2961            .await
2962        {
2963            Ok(response) => response,
2964            Err(Error::Http { status, body }) => {
2965                return Err(Error::QueryFailed(query_failure(status, body)));
2966            }
2967            Err(error) => return Err(error),
2968        };
2969
2970        let envelope = response
2971            .get("result_envelope")
2972            .filter(|envelope| !envelope.is_null())
2973            .ok_or_else(|| {
2974                Error::Codec(
2975                    "missing_payload_envelope: typed query result requires result_envelope"
2976                        .to_string(),
2977                )
2978            })?;
2979        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2980    }
2981
2982    async fn query_workflow_target<T: Serialize>(
2983        &self,
2984        workflow_id: &str,
2985        run_id: Option<&str>,
2986        query_name: &str,
2987        input: T,
2988    ) -> Result<Value> {
2989        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2990        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2991        let body = json!({
2992            "input": input_envelope
2993        });
2994        let path = match run_id {
2995            Some(run_id) => {
2996                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2997            }
2998            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2999        };
3000        let response: Value = match self
3001            .request_json(
3002                reqwest::Method::POST,
3003                &path,
3004                RequestProtocol::ControlPlane,
3005                Some(&body),
3006            )
3007            .await
3008        {
3009            Ok(response) => response,
3010            Err(Error::Http { status, body }) => {
3011                return Err(Error::QueryFailed(query_failure(status, body)));
3012            }
3013            Err(error) => return Err(error),
3014        };
3015
3016        if let Some(envelope) = response
3017            .get("result_envelope")
3018            .filter(|envelope| !envelope.is_null())
3019        {
3020            return decode_wire_value(envelope, DEFAULT_CODEC);
3021        }
3022
3023        Ok(response.get("result").cloned().unwrap_or(Value::Null))
3024    }
3025
3026    /// Send a synchronous update using fixed Avro Value arguments.
3027    pub async fn update_workflow<T: Serialize>(
3028        &self,
3029        workflow_id: &str,
3030        update_name: &str,
3031        input: T,
3032        request_id: Option<&str>,
3033    ) -> Result<Value> {
3034        let response = self
3035            .update_workflow_response(workflow_id, update_name, input, request_id)
3036            .await?;
3037        if let Some(envelope) = response
3038            .get("result_envelope")
3039            .filter(|envelope| !envelope.is_null())
3040        {
3041            return decode_wire_value(envelope, DEFAULT_CODEC);
3042        }
3043        Ok(response.get("result").cloned().unwrap_or(response))
3044    }
3045
3046    /// Send a synchronous update and retain a bytes-capable Avro result.
3047    pub async fn update_workflow_avro_value<T: Serialize>(
3048        &self,
3049        workflow_id: &str,
3050        update_name: &str,
3051        input: T,
3052        request_id: Option<&str>,
3053    ) -> Result<AvroValue> {
3054        let response = self
3055            .update_workflow_response(workflow_id, update_name, input, request_id)
3056            .await?;
3057        let envelope = response
3058            .get("result_envelope")
3059            .filter(|envelope| !envelope.is_null())
3060            .ok_or_else(|| {
3061                Error::Codec(
3062                    "missing_payload_envelope: typed update result requires result_envelope"
3063                        .to_string(),
3064                )
3065            })?;
3066        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3067    }
3068
3069    async fn update_workflow_response<T: Serialize>(
3070        &self,
3071        workflow_id: &str,
3072        update_name: &str,
3073        input: T,
3074        request_id: Option<&str>,
3075    ) -> Result<Value> {
3076        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3077        let mut body = json!({
3078            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3079            "wait_for": "completed",
3080        });
3081        if let Some(request_id) = request_id {
3082            body["request_id"] = json!(request_id);
3083        }
3084        self.request_json(
3085            reqwest::Method::POST,
3086            &format!("/workflows/{workflow_id}/update/{update_name}"),
3087            RequestProtocol::ControlPlane,
3088            Some(&body),
3089        )
3090        .await
3091    }
3092
3093    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3094        let path = format!("/workflows/{workflow_id}");
3095        let mut data: WorkflowDescription = self
3096            .request_json(
3097                reqwest::Method::GET,
3098                &path,
3099                RequestProtocol::ControlPlane,
3100                Option::<&Value>::None,
3101            )
3102            .await?;
3103        data.decode_payloads()?;
3104        Ok(data)
3105    }
3106
3107    /// Describe one selected run, including historical terminal runs.
3108    pub async fn describe_workflow_run(
3109        &self,
3110        workflow_id: &str,
3111        run_id: &str,
3112    ) -> Result<WorkflowDescription> {
3113        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3114        let mut data: WorkflowDescription = self
3115            .request_json(
3116                reqwest::Method::GET,
3117                &path,
3118                RequestProtocol::ControlPlane,
3119                Option::<&Value>::None,
3120            )
3121            .await?;
3122        data.decode_payloads()?;
3123        Ok(data)
3124    }
3125
3126    fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3127        let mut path = format!(
3128            "/workflows/{}/runs/{}/streams",
3129            percent_encode_path_segment(workflow_id),
3130            percent_encode_path_segment(run_id),
3131        );
3132        if let Some(stream_name) = stream_name {
3133            path.push('/');
3134            path.push_str(&percent_encode_path_segment(stream_name));
3135        }
3136        path
3137    }
3138
3139    /// List the run-scoped output streams already opened by a workflow.
3140    pub async fn list_workflow_streams(
3141        &self,
3142        workflow_id: &str,
3143        run_id: &str,
3144    ) -> Result<Vec<WorkflowStreamDescription>> {
3145        let response: WorkflowStreamListResponse = self
3146            .request_json(
3147                reqwest::Method::GET,
3148                &Self::workflow_stream_path(workflow_id, run_id, None),
3149                RequestProtocol::ControlPlane,
3150                Option::<&Value>::None,
3151            )
3152            .await?;
3153        Ok(response.streams)
3154    }
3155
3156    /// Describe stream lifecycle, offsets, pending count, and terminal error.
3157    pub async fn describe_workflow_stream(
3158        &self,
3159        workflow_id: &str,
3160        run_id: &str,
3161        stream_name: &str,
3162    ) -> Result<WorkflowStreamDescription> {
3163        let response: WorkflowStreamDescriptionResponse = self
3164            .request_json(
3165                reqwest::Method::GET,
3166                &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3167                RequestProtocol::ControlPlane,
3168                Option::<&Value>::None,
3169            )
3170            .await?;
3171        Ok(response.stream)
3172    }
3173
3174    /// Read one bounded page beginning at a zero-based offset.
3175    ///
3176    /// Delivery is at least once: persist `next_offset` only after processing
3177    /// the page. The future is cancellation-safe; dropping it cancels the
3178    /// in-flight request. Long polling is capped at 60 seconds by the SDK and
3179    /// service contract.
3180    pub async fn subscribe_workflow_stream(
3181        &self,
3182        workflow_id: &str,
3183        run_id: &str,
3184        stream_name: &str,
3185        from_offset: u64,
3186        max_items: usize,
3187        wait: Duration,
3188    ) -> Result<WorkflowStreamPage> {
3189        let max_items = max_items.clamp(1, 500);
3190        let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3191        let path = format!(
3192            "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3193            Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3194        );
3195        let response: WorkflowStreamPageResponse = self
3196            .request_json_with_timeout(
3197                reqwest::Method::GET,
3198                &path,
3199                RequestProtocol::ControlPlane,
3200                Option::<&Value>::None,
3201                Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3202            )
3203            .await?;
3204
3205        let items = response
3206            .items
3207            .into_iter()
3208            .map(|raw| {
3209                let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3210                let envelope = raw.get("payload").cloned();
3211                let payload = envelope
3212                    .as_ref()
3213                    .filter(|value| value.get("blob").is_some())
3214                    .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3215                    .transpose()?
3216                    .map(AvroValue::into_json)
3217                    .transpose()?;
3218                Ok(WorkflowStreamItem {
3219                    offset,
3220                    payload,
3221                    payload_envelope: envelope,
3222                    payload_reference: raw
3223                        .get("payload_reference")
3224                        .and_then(Value::as_str)
3225                        .map(str::to_string),
3226                    payload_codec: raw
3227                        .get("payload_codec")
3228                        .and_then(Value::as_str)
3229                        .map(str::to_string),
3230                    idempotency_key: raw
3231                        .get("idempotency_key")
3232                        .and_then(Value::as_str)
3233                        .map(str::to_string),
3234                    item_type: raw
3235                        .get("item_type")
3236                        .and_then(Value::as_str)
3237                        .map(str::to_string),
3238                    content_type: raw
3239                        .get("content_type")
3240                        .and_then(Value::as_str)
3241                        .map(str::to_string),
3242                    origin: raw
3243                        .get("origin")
3244                        .and_then(Value::as_str)
3245                        .map(str::to_string),
3246                    origin_reference: raw
3247                        .get("origin_reference")
3248                        .and_then(Value::as_str)
3249                        .map(str::to_string),
3250                    emitted_at: raw
3251                        .get("emitted_at")
3252                        .and_then(Value::as_str)
3253                        .map(str::to_string),
3254                    raw,
3255                })
3256            })
3257            .collect::<Result<Vec<_>>>()?;
3258        Ok(WorkflowStreamPage {
3259            stream: response.stream,
3260            items,
3261            next_offset: response.next_offset,
3262            terminal: response.terminal,
3263        })
3264    }
3265
3266    /// Append inline Avro envelopes or opaque external payload references.
3267    pub async fn append_workflow_stream(
3268        &self,
3269        workflow_id: &str,
3270        run_id: &str,
3271        stream_name: &str,
3272        items: &[WorkflowStreamAppendItem],
3273        max_pending_items: Option<u64>,
3274    ) -> Result<WorkflowStreamAppendResult> {
3275        if items.is_empty() {
3276            return Err(Error::Codec(
3277                "workflow_stream_items_empty: append requires at least one item".to_string(),
3278            ));
3279        }
3280        let mut body = json!({
3281            "items": items
3282                .iter()
3283                .map(|item| item.wire_value(None))
3284                .collect::<Vec<_>>(),
3285        });
3286        if let Some(max_pending_items) = max_pending_items {
3287            if max_pending_items == 0 {
3288                return Err(Error::Codec(
3289                    "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3290                        .to_string(),
3291                ));
3292            }
3293            body["max_pending_items"] = json!(max_pending_items);
3294        }
3295        let response: WorkflowStreamAppendResponse = self
3296            .request_json(
3297                reqwest::Method::POST,
3298                &format!(
3299                    "{}/items",
3300                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3301                ),
3302                RequestProtocol::ControlPlane,
3303                Some(&body),
3304            )
3305            .await?;
3306        Ok(WorkflowStreamAppendResult {
3307            stream: response.stream,
3308            accepted_offsets: response.accepted_offsets,
3309            accepted: response.accepted,
3310            deduped: response.deduped,
3311        })
3312    }
3313
3314    /// Close a stream, or mark it errored when `error_reason` is supplied.
3315    pub async fn close_workflow_stream(
3316        &self,
3317        workflow_id: &str,
3318        run_id: &str,
3319        stream_name: &str,
3320        error_reason: Option<&str>,
3321        retention_seconds: Option<u64>,
3322    ) -> Result<WorkflowStreamDescription> {
3323        let mut body = json!({});
3324        if let Some(error_reason) = error_reason {
3325            body["error_reason"] = json!(error_reason);
3326        }
3327        if let Some(retention_seconds) = retention_seconds {
3328            if retention_seconds == 0 {
3329                return Err(Error::Codec(
3330                    "workflow_stream_retention_invalid: retention_seconds must be positive"
3331                        .to_string(),
3332                ));
3333            }
3334            body["retention_seconds"] = json!(retention_seconds);
3335        }
3336        let response: WorkflowStreamDescriptionResponse = self
3337            .request_json(
3338                reqwest::Method::POST,
3339                &format!(
3340                    "{}/close",
3341                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3342                ),
3343                RequestProtocol::ControlPlane,
3344                Some(&body),
3345            )
3346            .await?;
3347        Ok(response.stream)
3348    }
3349
3350    pub async fn register_worker(
3351        &self,
3352        worker_id: &str,
3353        task_queue: &str,
3354        supported_workflow_types: Vec<String>,
3355        supported_activity_types: Vec<String>,
3356        max_concurrent_workflow_tasks: usize,
3357        max_concurrent_activity_tasks: usize,
3358    ) -> Result<RegisterWorkerResponse> {
3359        self.register_worker_with_capabilities(
3360            worker_id,
3361            task_queue,
3362            supported_workflow_types,
3363            supported_activity_types,
3364            max_concurrent_workflow_tasks,
3365            max_concurrent_activity_tasks,
3366            Vec::new(),
3367        )
3368        .await
3369    }
3370
3371    /// Register a worker and explicitly advertise additive worker capabilities.
3372    pub async fn register_worker_with_capabilities(
3373        &self,
3374        worker_id: &str,
3375        task_queue: &str,
3376        supported_workflow_types: Vec<String>,
3377        supported_activity_types: Vec<String>,
3378        max_concurrent_workflow_tasks: usize,
3379        max_concurrent_activity_tasks: usize,
3380        capabilities: Vec<String>,
3381    ) -> Result<RegisterWorkerResponse> {
3382        self.register_worker_with_command_contracts(
3383            worker_id,
3384            task_queue,
3385            supported_workflow_types,
3386            supported_activity_types,
3387            max_concurrent_workflow_tasks,
3388            max_concurrent_activity_tasks,
3389            capabilities,
3390            Value::Object(serde_json::Map::new()),
3391        )
3392        .await
3393    }
3394
3395    /// Register a worker and advertise its named query and update handlers.
3396    ///
3397    /// This Rust SDK cannot execute synchronous pre-accept update validation,
3398    /// so a workflow contract with a non-empty or malformed
3399    /// `update_validators` declaration returns
3400    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
3401    #[allow(clippy::too_many_arguments)]
3402    pub async fn register_worker_with_command_contracts(
3403        &self,
3404        worker_id: &str,
3405        task_queue: &str,
3406        supported_workflow_types: Vec<String>,
3407        supported_activity_types: Vec<String>,
3408        max_concurrent_workflow_tasks: usize,
3409        max_concurrent_activity_tasks: usize,
3410        capabilities: Vec<String>,
3411        workflow_command_contracts: Value,
3412    ) -> Result<RegisterWorkerResponse> {
3413        if let Some(contracts) = workflow_command_contracts.as_object() {
3414            for (workflow_type, contract) in contracts {
3415                let Some(update_validators) = contract.get("update_validators") else {
3416                    continue;
3417                };
3418                if !update_validators
3419                    .as_array()
3420                    .is_some_and(|validators| validators.is_empty())
3421                {
3422                    return Err(Error::UnsupportedUpdateValidators {
3423                        workflow_type: workflow_type.clone(),
3424                    });
3425                }
3426            }
3427        }
3428
3429        let mut body = json!({
3430            "worker_id": worker_id,
3431            "task_queue": task_queue,
3432            "runtime": "rust",
3433            "sdk_version": SDK_VERSION,
3434            "supported_workflow_types": supported_workflow_types,
3435            "supported_activity_types": supported_activity_types,
3436            "capabilities": capabilities,
3437            "capability_manifest": portable_worker_affinity_capability_manifest(),
3438            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3439            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3440        });
3441        if workflow_command_contracts
3442            .as_object()
3443            .is_some_and(|contracts| !contracts.is_empty())
3444        {
3445            body["workflow_command_contracts"] = workflow_command_contracts;
3446        }
3447
3448        self.request_json(
3449            reqwest::Method::POST,
3450            "/worker/register",
3451            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3452            Some(&body),
3453        )
3454        .await
3455    }
3456
3457    /// Gracefully remove one worker's registration through the worker plane.
3458    ///
3459    /// This operation is separate from operator-facing worker management. It
3460    /// uses worker-protocol authentication and returns the server's lease
3461    /// recovery result.
3462    pub async fn deregister_worker_registration(
3463        &self,
3464        worker_id: &str,
3465    ) -> Result<WorkerDeregistrationEnvelope> {
3466        let path = format!(
3467            "/worker/registrations/{}",
3468            percent_encode_path_segment(worker_id)
3469        );
3470        self.request_json(
3471            reqwest::Method::DELETE,
3472            &path,
3473            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3474            Option::<&Value>::None,
3475        )
3476        .await
3477    }
3478
3479    /// Long-poll for an ephemeral, read-only workflow query task.
3480    pub async fn poll_query_task(
3481        &self,
3482        worker_id: &str,
3483        task_queue: &str,
3484        timeout: Duration,
3485    ) -> Result<Option<QueryTask>> {
3486        Ok(self
3487            .poll_query_task_response(worker_id, task_queue, timeout)
3488            .await?
3489            .task)
3490    }
3491
3492    /// Poll a query task while preserving server stop and drain metadata.
3493    pub async fn poll_query_task_response(
3494        &self,
3495        worker_id: &str,
3496        task_queue: &str,
3497        timeout: Duration,
3498    ) -> Result<PollQueryTaskResponse> {
3499        let poll_request_id = unique_request_id("rust-query-poll");
3500        self.poll_query_task_response_with_request_id(
3501            worker_id,
3502            task_queue,
3503            timeout,
3504            &poll_request_id,
3505            1,
3506        )
3507        .await
3508    }
3509
3510    async fn poll_query_task_response_with_request_id(
3511        &self,
3512        worker_id: &str,
3513        task_queue: &str,
3514        timeout: Duration,
3515        poll_request_id: &str,
3516        transport_retries: usize,
3517    ) -> Result<PollQueryTaskResponse> {
3518        let timeout_seconds = long_poll_timeout_seconds(timeout);
3519        let body = json!({
3520            "worker_id": worker_id,
3521            "task_queue": task_queue,
3522            "poll_request_id": poll_request_id,
3523            "timeout_seconds": timeout_seconds,
3524        });
3525        self.poll_request_json(
3526            "/worker/query-tasks/poll",
3527            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3528            &body,
3529            timeout + Duration::from_secs(5),
3530            transport_retries,
3531        )
3532        .await
3533    }
3534
3535    /// Complete a query task without appending workflow history.
3536    pub async fn complete_query_task<T: Serialize>(
3537        &self,
3538        query_task_id: &str,
3539        lease_owner: &str,
3540        query_task_attempt: u64,
3541        result: T,
3542        codec: &str,
3543    ) -> Result<Value> {
3544        let typed_result = AvroValue::from_serialize(&result)?;
3545        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3546        self.complete_query_task_with_envelope(
3547            query_task_id,
3548            lease_owner,
3549            query_task_attempt,
3550            typed_result.into_json()?,
3551            result_envelope,
3552        )
3553        .await
3554    }
3555
3556    async fn complete_query_task_with_envelope(
3557        &self,
3558        query_task_id: &str,
3559        lease_owner: &str,
3560        query_task_attempt: u64,
3561        result: Value,
3562        result_envelope: Value,
3563    ) -> Result<Value> {
3564        let body = json!({
3565            "lease_owner": lease_owner,
3566            "query_task_attempt": query_task_attempt,
3567            "result": result,
3568            "result_envelope": result_envelope,
3569        });
3570        let path = format!("/worker/query-tasks/{query_task_id}/complete");
3571        let response = self
3572            .request_json(
3573                reqwest::Method::POST,
3574                &path,
3575                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3576                Some(&body),
3577            )
3578            .await;
3579        query_task_response(response)
3580    }
3581
3582    /// Report a stable machine-readable query-task failure.
3583    pub async fn fail_query_task(
3584        &self,
3585        query_task_id: &str,
3586        lease_owner: &str,
3587        query_task_attempt: u64,
3588        message: impl Into<String>,
3589        reason: impl Into<String>,
3590        failure_type: impl Into<String>,
3591    ) -> Result<Value> {
3592        let body = json!({
3593            "lease_owner": lease_owner,
3594            "query_task_attempt": query_task_attempt,
3595            "failure": {
3596                "message": message.into(),
3597                "reason": reason.into(),
3598                "type": failure_type.into(),
3599            }
3600        });
3601        let path = format!("/worker/query-tasks/{query_task_id}/fail");
3602        let response = self
3603            .request_json(
3604                reqwest::Method::POST,
3605                &path,
3606                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3607                Some(&body),
3608            )
3609            .await;
3610        query_task_response(response)
3611    }
3612
3613    pub async fn heartbeat_worker(
3614        &self,
3615        worker_id: &str,
3616        workflow_available: usize,
3617        activity_available: usize,
3618    ) -> Result<Value> {
3619        let body = json!({
3620            "worker_id": worker_id,
3621            "task_slots": {
3622                "workflow_available": workflow_available,
3623                "activity_available": activity_available
3624            },
3625            "process_metrics": {
3626                "process_id": std::process::id(),
3627                "process_uptime_seconds": 0
3628            }
3629        });
3630
3631        self.request_json(
3632            reqwest::Method::POST,
3633            "/worker/heartbeat",
3634            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3635            Some(&body),
3636        )
3637        .await
3638    }
3639
3640    pub async fn poll_workflow_task(
3641        &self,
3642        worker_id: &str,
3643        task_queue: &str,
3644        timeout: Duration,
3645    ) -> Result<Option<WorkflowTask>> {
3646        Ok(self
3647            .poll_workflow_task_response(worker_id, task_queue, timeout)
3648            .await?
3649            .task)
3650    }
3651
3652    pub async fn poll_workflow_task_response(
3653        &self,
3654        worker_id: &str,
3655        task_queue: &str,
3656        timeout: Duration,
3657    ) -> Result<PollWorkflowTaskResponse> {
3658        let poll_request_id = unique_request_id("rust-workflow-poll");
3659        self.poll_workflow_task_response_with_request_id(
3660            worker_id,
3661            task_queue,
3662            timeout,
3663            &poll_request_id,
3664            1,
3665        )
3666        .await
3667    }
3668
3669    async fn poll_workflow_task_response_with_request_id(
3670        &self,
3671        worker_id: &str,
3672        task_queue: &str,
3673        timeout: Duration,
3674        poll_request_id: &str,
3675        transport_retries: usize,
3676    ) -> Result<PollWorkflowTaskResponse> {
3677        let body = json!({
3678            "worker_id": worker_id,
3679            "task_queue": task_queue,
3680            "poll_request_id": poll_request_id,
3681            "timeout_seconds": long_poll_timeout_seconds(timeout),
3682        });
3683        let mut data: PollWorkflowTaskResponse = self
3684            .poll_request_json(
3685                "/worker/workflow-tasks/poll",
3686                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3687                &body,
3688                timeout + Duration::from_secs(5),
3689                transport_retries,
3690            )
3691            .await?;
3692
3693        if let Some(task) = data.task.as_mut() {
3694            self.fetch_remaining_workflow_history(worker_id, task)
3695                .await?;
3696        }
3697
3698        Ok(data)
3699    }
3700
3701    async fn fetch_remaining_workflow_history(
3702        &self,
3703        worker_id: &str,
3704        task: &mut WorkflowTask,
3705    ) -> Result<()> {
3706        let mut next_token = task.next_history_page_token.clone();
3707
3708        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3709            let lease_owner = task
3710                .lease_owner
3711                .clone()
3712                .unwrap_or_else(|| worker_id.to_string());
3713            let page = self
3714                .workflow_task_history_page(
3715                    &task.task_id,
3716                    &lease_owner,
3717                    task.workflow_task_attempt,
3718                    &token,
3719                )
3720                .await?;
3721
3722            task.append_history_page(page);
3723
3724            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3725                return Err(Error::Codec(
3726                    "workflow history pagination returned the same page token".to_string(),
3727                ));
3728            }
3729
3730            next_token = task.next_history_page_token.clone();
3731        }
3732
3733        Ok(())
3734    }
3735
3736    async fn workflow_task_history_page(
3737        &self,
3738        task_id: &str,
3739        lease_owner: &str,
3740        workflow_task_attempt: u64,
3741        next_history_page_token: &str,
3742    ) -> Result<WorkflowTaskHistoryPage> {
3743        let body = json!({
3744            "lease_owner": lease_owner,
3745            "workflow_task_attempt": workflow_task_attempt,
3746            "next_history_page_token": next_history_page_token
3747        });
3748        let path = format!("/worker/workflow-tasks/{task_id}/history");
3749
3750        self.request_json(
3751            reqwest::Method::POST,
3752            &path,
3753            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3754            Some(&body),
3755        )
3756        .await
3757    }
3758
3759    pub async fn complete_workflow_task(
3760        &self,
3761        task_id: &str,
3762        lease_owner: &str,
3763        workflow_task_attempt: u64,
3764        commands: Vec<Value>,
3765    ) -> Result<Value> {
3766        self.complete_workflow_task_with_message_streams(
3767            task_id,
3768            lease_owner,
3769            workflow_task_attempt,
3770            commands,
3771            Vec::new(),
3772            Vec::new(),
3773        )
3774        .await
3775    }
3776
3777    async fn complete_workflow_task_with_message_streams(
3778        &self,
3779        task_id: &str,
3780        lease_owner: &str,
3781        workflow_task_attempt: u64,
3782        commands: Vec<Value>,
3783        message_stream_cursors: Vec<Value>,
3784        message_stream_waits: Vec<Value>,
3785    ) -> Result<Value> {
3786        validate_workflow_task_commands(&commands)?;
3787        let has_message_stream_metadata =
3788            !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3789        if has_message_stream_metadata
3790            && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3791        {
3792            return Err(Error::Codec(
3793                "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3794                    .to_string(),
3795            ));
3796        }
3797        let protocol_version = workflow_completion_protocol_version_with_message_streams(
3798            &commands,
3799            has_message_stream_metadata,
3800        );
3801        let mut body = json!({
3802            "lease_owner": lease_owner,
3803            "workflow_task_attempt": workflow_task_attempt,
3804            "commands": commands
3805        });
3806        if !message_stream_cursors.is_empty() {
3807            body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3808        }
3809        if !message_stream_waits.is_empty() {
3810            body["message_stream_waits"] = Value::Array(message_stream_waits);
3811        }
3812        let path = format!("/worker/workflow-tasks/{task_id}/complete");
3813        self.request_json(
3814            reqwest::Method::POST,
3815            &path,
3816            RequestProtocol::Worker(protocol_version),
3817            Some(&body),
3818        )
3819        .await
3820    }
3821
3822    pub async fn fail_workflow_task(
3823        &self,
3824        task_id: &str,
3825        lease_owner: &str,
3826        workflow_task_attempt: u64,
3827        message: impl Into<String>,
3828    ) -> Result<Value> {
3829        self.fail_workflow_task_with_type(
3830            task_id,
3831            lease_owner,
3832            workflow_task_attempt,
3833            message,
3834            "RustWorkflowTaskFailure",
3835        )
3836        .await
3837    }
3838
3839    async fn fail_workflow_task_with_type(
3840        &self,
3841        task_id: &str,
3842        lease_owner: &str,
3843        workflow_task_attempt: u64,
3844        message: impl Into<String>,
3845        failure_type: &str,
3846    ) -> Result<Value> {
3847        let body = json!({
3848            "lease_owner": lease_owner,
3849            "workflow_task_attempt": workflow_task_attempt,
3850            "failure": {
3851                "message": message.into(),
3852                "type": failure_type
3853            }
3854        });
3855        let path = format!("/worker/workflow-tasks/{task_id}/fail");
3856        self.request_json(
3857            reqwest::Method::POST,
3858            &path,
3859            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3860            Some(&body),
3861        )
3862        .await
3863    }
3864
3865    pub async fn poll_activity_task(
3866        &self,
3867        worker_id: &str,
3868        task_queue: &str,
3869        timeout: Duration,
3870    ) -> Result<Option<ActivityTask>> {
3871        Ok(self
3872            .poll_activity_task_response(worker_id, task_queue, timeout)
3873            .await?
3874            .task)
3875    }
3876
3877    /// Poll an activity task while preserving server stop and drain metadata.
3878    pub async fn poll_activity_task_response(
3879        &self,
3880        worker_id: &str,
3881        task_queue: &str,
3882        timeout: Duration,
3883    ) -> Result<PollActivityTaskResponse> {
3884        let poll_request_id = unique_request_id("rust-activity-poll");
3885        self.poll_activity_task_response_with_request_id(
3886            worker_id,
3887            task_queue,
3888            timeout,
3889            &poll_request_id,
3890            1,
3891        )
3892        .await
3893    }
3894
3895    async fn poll_activity_task_response_with_request_id(
3896        &self,
3897        worker_id: &str,
3898        task_queue: &str,
3899        timeout: Duration,
3900        poll_request_id: &str,
3901        transport_retries: usize,
3902    ) -> Result<PollActivityTaskResponse> {
3903        let body = json!({
3904            "worker_id": worker_id,
3905            "task_queue": task_queue,
3906            "poll_request_id": poll_request_id,
3907            "timeout_seconds": long_poll_timeout_seconds(timeout),
3908        });
3909        let data: PollActivityTaskResponse = self
3910            .poll_request_json(
3911                "/worker/activity-tasks/poll",
3912                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3913                &body,
3914                timeout + Duration::from_secs(5),
3915                transport_retries,
3916            )
3917            .await?;
3918        Ok(data)
3919    }
3920
3921    pub async fn complete_activity_task<T: Serialize>(
3922        &self,
3923        task_id: &str,
3924        activity_attempt_id: &str,
3925        lease_owner: &str,
3926        result: T,
3927        codec: &str,
3928    ) -> Result<Value> {
3929        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3930        let body = json!({
3931            "activity_attempt_id": activity_attempt_id,
3932            "lease_owner": lease_owner,
3933            "result": result
3934        });
3935        let path = format!("/worker/activity-tasks/{task_id}/complete");
3936        activity_task_response(
3937            self.request_json(
3938                reqwest::Method::POST,
3939                &path,
3940                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3941                Some(&body),
3942            )
3943            .await,
3944            "complete",
3945            task_id,
3946            activity_attempt_id,
3947        )
3948    }
3949
3950    pub async fn fail_activity_task(
3951        &self,
3952        task_id: &str,
3953        activity_attempt_id: &str,
3954        lease_owner: &str,
3955        message: impl Into<String>,
3956        non_retryable: bool,
3957    ) -> Result<Value> {
3958        let body = json!({
3959            "activity_attempt_id": activity_attempt_id,
3960            "lease_owner": lease_owner,
3961            "failure": {
3962                "message": message.into(),
3963                "type": "RustActivityFailure",
3964                "non_retryable": non_retryable
3965            }
3966        });
3967        let path = format!("/worker/activity-tasks/{task_id}/fail");
3968        activity_task_response(
3969            self.request_json(
3970                reqwest::Method::POST,
3971                &path,
3972                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3973                Some(&body),
3974            )
3975            .await,
3976            "fail",
3977            task_id,
3978            activity_attempt_id,
3979        )
3980    }
3981
3982    pub async fn heartbeat_activity_task<T: Serialize>(
3983        &self,
3984        task_id: &str,
3985        activity_attempt_id: &str,
3986        lease_owner: &str,
3987        details: T,
3988    ) -> Result<ActivityHeartbeatResponse> {
3989        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3990        let body = json!({
3991            "activity_attempt_id": activity_attempt_id,
3992            "lease_owner": lease_owner,
3993            "details": details
3994        });
3995        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
3996        activity_task_response(
3997            self.request_json(
3998                reqwest::Method::POST,
3999                &path,
4000                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4001                Some(&body),
4002            )
4003            .await,
4004            "heartbeat",
4005            task_id,
4006            activity_attempt_id,
4007        )
4008    }
4009
4010    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4011        &self,
4012        method: reqwest::Method,
4013        path: &str,
4014        protocol: RequestProtocol,
4015        body: Option<&B>,
4016    ) -> Result<T> {
4017        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4018            .await
4019    }
4020
4021    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4022        &self,
4023        method: reqwest::Method,
4024        path: &str,
4025        protocol: RequestProtocol,
4026        body: Option<&B>,
4027        timeout: Duration,
4028    ) -> Result<T> {
4029        let auth_token = self.auth_token(protocol)?;
4030        let mut request = self
4031            .http
4032            .request(method, format!("{}/api{}", self.base_url, path))
4033            .timeout(timeout)
4034            .header(reqwest::header::ACCEPT, "application/json")
4035            .header(reqwest::header::CONTENT_TYPE, "application/json")
4036            .header("X-Namespace", &self.namespace);
4037
4038        match protocol {
4039            RequestProtocol::Worker(version) => {
4040                request = request.header("X-Durable-Workflow-Protocol-Version", version);
4041            }
4042            RequestProtocol::ControlPlane => {
4043                request = request.header(
4044                    "X-Durable-Workflow-Control-Plane-Version",
4045                    CONTROL_PLANE_VERSION,
4046                );
4047            }
4048        }
4049
4050        if let Some(token) = auth_token {
4051            request = request.bearer_auth(token);
4052        }
4053
4054        if let Some(body) = body {
4055            request = request.json(body);
4056        }
4057
4058        let response = request.send().await?;
4059        let status = response.status();
4060        let bytes = response.bytes().await?;
4061
4062        if !status.is_success() {
4063            let body = String::from_utf8_lossy(&bytes).to_string();
4064            if let Some(protocol) = protocol_failure(status, &body) {
4065                return Err(Error::Protocol(protocol));
4066            }
4067            return Err(Error::Http { status, body });
4068        }
4069
4070        if bytes.is_empty() {
4071            return Ok(serde_json::from_value(Value::Null)?);
4072        }
4073
4074        Ok(serde_json::from_slice(&bytes)?)
4075    }
4076
4077    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4078        &self,
4079        path: &str,
4080        protocol: RequestProtocol,
4081        body: &B,
4082        timeout: Duration,
4083        max_retries: usize,
4084    ) -> Result<T> {
4085        let mut retries = 0;
4086
4087        loop {
4088            let response = self
4089                .request_json_with_timeout(
4090                    reqwest::Method::POST,
4091                    path,
4092                    protocol,
4093                    Some(body),
4094                    timeout,
4095                )
4096                .await;
4097
4098            match response {
4099                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4100                response => return worker_poll_response(response),
4101            }
4102        }
4103    }
4104
4105    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4106        match protocol {
4107            RequestProtocol::Worker(_) => {
4108                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4109                    return Ok(Some(token));
4110                }
4111                if self.control_token.is_some() {
4112                    return Err(Error::MissingRoleCredentials {
4113                        role: "worker",
4114                        opposite_role: "control",
4115                    });
4116                }
4117                Ok(None)
4118            }
4119            RequestProtocol::ControlPlane => {
4120                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4121                    return Ok(Some(token));
4122                }
4123                if self.worker_token.is_some() {
4124                    return Err(Error::MissingRoleCredentials {
4125                        role: "control",
4126                        opposite_role: "worker",
4127                    });
4128                }
4129                Ok(None)
4130            }
4131        }
4132    }
4133}
4134
4135fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4136    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4137    let reason = body
4138        .get("reason")
4139        .and_then(Value::as_str)
4140        .unwrap_or("query_rejected")
4141        .to_string();
4142    let message = body
4143        .get("message")
4144        .or_else(|| body.get("error"))
4145        .and_then(Value::as_str)
4146        .unwrap_or("workflow query was rejected")
4147        .to_string();
4148
4149    QueryFailure {
4150        status: status.as_u16(),
4151        reason,
4152        message,
4153        body,
4154    }
4155}
4156
4157fn workflow_command_result(
4158    command: WorkflowCommandKind,
4159    data: Value,
4160    workflow_id: &str,
4161    run_id: Option<&str>,
4162) -> WorkflowCommandResult {
4163    WorkflowCommandResult {
4164        command,
4165        workflow_id: data
4166            .get("workflow_id")
4167            .and_then(Value::as_str)
4168            .unwrap_or(workflow_id)
4169            .to_string(),
4170        run_id: data
4171            .get("run_id")
4172            .and_then(Value::as_str)
4173            .or(run_id)
4174            .map(str::to_string),
4175        outcome: data
4176            .get("outcome")
4177            .and_then(Value::as_str)
4178            .map(str::to_string),
4179        reason: data
4180            .get("reason")
4181            .and_then(Value::as_str)
4182            .map(str::to_string),
4183        command_status: data
4184            .get("command_status")
4185            .and_then(Value::as_str)
4186            .map(str::to_string),
4187        raw: data,
4188    }
4189}
4190
4191fn workflow_command_rejection(
4192    command: WorkflowCommandKind,
4193    status: reqwest::StatusCode,
4194    raw_body: String,
4195    workflow_id: &str,
4196    run_id: Option<&str>,
4197) -> WorkflowCommandRejection {
4198    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4199    WorkflowCommandRejection {
4200        command,
4201        status: status.as_u16(),
4202        reason: body
4203            .get("reason")
4204            .and_then(Value::as_str)
4205            .unwrap_or("workflow_command_rejected")
4206            .to_string(),
4207        message: body
4208            .get("message")
4209            .or_else(|| body.get("error"))
4210            .and_then(Value::as_str)
4211            .unwrap_or("workflow lifecycle command was rejected")
4212            .to_string(),
4213        workflow_id: body
4214            .get("workflow_id")
4215            .and_then(Value::as_str)
4216            .unwrap_or(workflow_id)
4217            .to_string(),
4218        run_id: body
4219            .get("run_id")
4220            .and_then(Value::as_str)
4221            .or(run_id)
4222            .map(str::to_string),
4223        target_scope: body
4224            .get("target_scope")
4225            .and_then(Value::as_str)
4226            .map(str::to_string),
4227        body,
4228    }
4229}
4230
4231fn query_task_response(response: Result<Value>) -> Result<Value> {
4232    match response {
4233        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4234        response => response,
4235    }
4236}
4237
4238fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4239    match response {
4240        Err(Error::Http { status, body })
4241            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4242        {
4243            Ok(serde_json::from_str(&body)?)
4244        }
4245        response => response,
4246    }
4247}
4248
4249fn worker_poll_body_is_stop(body: &str) -> bool {
4250    serde_json::from_str::<Value>(body)
4251        .ok()
4252        .is_some_and(|body| {
4253            worker_poll_is_stop(
4254                body.get("poll_status").and_then(Value::as_str),
4255                body.get("reason").and_then(Value::as_str),
4256            )
4257        })
4258}
4259
4260fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4261    matches!(poll_status, Some("draining" | "stopped"))
4262        || matches!(reason, Some("worker_draining" | "worker_stopped"))
4263}
4264
4265fn query_task_rejection_is_final(error: &Error) -> bool {
4266    matches!(
4267        error,
4268        Error::QueryFailed(failure)
4269            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4270    )
4271}
4272
4273fn activity_task_response<T>(
4274    response: Result<T>,
4275    operation: &str,
4276    task_id: &str,
4277    activity_attempt_id: &str,
4278) -> Result<T> {
4279    match response {
4280        Err(Error::Http { status, body }) => {
4281            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4282            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4283                operation: operation.to_string(),
4284                status: status.as_u16(),
4285                reason: body
4286                    .get("reason")
4287                    .and_then(Value::as_str)
4288                    .unwrap_or("activity_task_rejected")
4289                    .to_string(),
4290                task_id: body
4291                    .get("task_id")
4292                    .and_then(Value::as_str)
4293                    .unwrap_or(task_id)
4294                    .to_string(),
4295                activity_attempt_id: body
4296                    .get("activity_attempt_id")
4297                    .and_then(Value::as_str)
4298                    .unwrap_or(activity_attempt_id)
4299                    .to_string(),
4300                cancel_requested: body
4301                    .get("cancel_requested")
4302                    .and_then(Value::as_bool)
4303                    .unwrap_or(false),
4304                can_continue: body.get("can_continue").and_then(Value::as_bool),
4305                run_closed_reason: body
4306                    .get("run_closed_reason")
4307                    .and_then(Value::as_str)
4308                    .map(str::to_string),
4309                body,
4310            }))
4311        }
4312        response => response,
4313    }
4314}
4315
4316fn activity_task_rejection_is_final(error: &Error) -> bool {
4317    matches!(
4318        error,
4319        Error::ActivityTaskRejected(rejection)
4320            if matches!(
4321                rejection.reason.as_str(),
4322                "run_cancelled"
4323                    | "run_terminated"
4324                    | "attempt_closed"
4325                    | "stale_attempt"
4326                    | "activity_cancelled"
4327                    | "task_cancelled"
4328                    | "run_closed"
4329                    | "activity_not_running"
4330                    | "attempt_not_found"
4331            )
4332    )
4333}
4334
4335fn workflow_task_completion_is_terminal_timeout(
4336    error: &Error,
4337    task_id: &str,
4338    workflow_task_attempt: u64,
4339    run_id: Option<&str>,
4340) -> bool {
4341    let Error::Http { status, body } = error else {
4342        return false;
4343    };
4344    if *status != reqwest::StatusCode::CONFLICT {
4345        return false;
4346    }
4347
4348    let Some(run_id) = run_id else {
4349        return false;
4350    };
4351    let Ok(body) = serde_json::from_str::<Value>(body) else {
4352        return false;
4353    };
4354
4355    body.get("recorded").and_then(Value::as_bool) == Some(false)
4356        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4357        && body.get("run_status").and_then(Value::as_str) == Some("failed")
4358        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4359        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4360        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4361}
4362
4363fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4364    let body: Value = serde_json::from_str(raw_body).ok()?;
4365    let reason = body.get("reason")?.as_str()?;
4366    if !matches!(
4367        reason,
4368        "missing_protocol_version"
4369            | "unsupported_protocol_version"
4370            | "missing_control_plane_version"
4371            | "unsupported_control_plane_version"
4372    ) {
4373        return None;
4374    }
4375
4376    Some(ProtocolFailure {
4377        status: status.as_u16(),
4378        reason: reason.to_string(),
4379        message: body
4380            .get("message")
4381            .or_else(|| body.get("error"))
4382            .and_then(Value::as_str)
4383            .unwrap_or("protocol version rejected")
4384            .to_string(),
4385        supported_version: body
4386            .get("supported_version")
4387            .and_then(Value::as_str)
4388            .map(str::to_string),
4389        requested_version: body
4390            .get("requested_version")
4391            .and_then(Value::as_str)
4392            .map(str::to_string),
4393        body,
4394    })
4395}
4396
4397fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4398    timeout
4399        .as_secs()
4400        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4401        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4402}
4403
4404fn worker_operation_is_retryable(error: &Error) -> bool {
4405    if worker_poll_capacity_retry_after(error).is_some()
4406        || worker_operation_is_explicitly_non_retryable(error)
4407    {
4408        return false;
4409    }
4410
4411    match error {
4412        Error::Transport(error) => {
4413            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4414        }
4415        Error::Http { status, .. } => {
4416            matches!(
4417                *status,
4418                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4419            ) || status.is_server_error()
4420        }
4421        _ => false,
4422    }
4423}
4424
4425fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4426    let Error::Http { body, .. } = error else {
4427        return false;
4428    };
4429
4430    serde_json::from_str::<Value>(body)
4431        .ok()
4432        .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4433        == Some(false)
4434}
4435
4436fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4437    let Error::Http { status, body } = error else {
4438        return None;
4439    };
4440    if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4441        return None;
4442    }
4443
4444    let body = serde_json::from_str::<Value>(body).ok()?;
4445    let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4446        == Some("long_poll_capacity_exhausted")
4447        || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4448    if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4449        return None;
4450    }
4451
4452    Some(Duration::from_secs(
4453        body.get("retry_after_seconds")
4454            .and_then(Value::as_u64)
4455            .unwrap_or_default(),
4456    ))
4457}
4458
4459fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4460    let exponent = retry.saturating_sub(1).min(31) as u32;
4461    policy
4462        .initial_backoff
4463        .saturating_mul(1_u32 << exponent)
4464        .min(policy.max_backoff)
4465}
4466
4467#[derive(Debug)]
4468pub struct ClientBuilder {
4469    base_url: String,
4470    token: Option<String>,
4471    control_token: Option<String>,
4472    worker_token: Option<String>,
4473    namespace: String,
4474    timeout: Duration,
4475}
4476
4477impl ClientBuilder {
4478    pub fn token(mut self, token: Option<String>) -> Self {
4479        self.token = token;
4480        self
4481    }
4482
4483    pub fn control_token(mut self, token: Option<String>) -> Self {
4484        self.control_token = token;
4485        self
4486    }
4487
4488    pub fn worker_token(mut self, token: Option<String>) -> Self {
4489        self.worker_token = token;
4490        self
4491    }
4492
4493    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4494        self.namespace = namespace.into();
4495        self
4496    }
4497
4498    pub fn timeout(mut self, timeout: Duration) -> Self {
4499        self.timeout = timeout;
4500        self
4501    }
4502
4503    pub fn build(self) -> Result<Client> {
4504        let base_url = self.base_url.trim_end_matches('/').to_string();
4505        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4506            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4507            .unwrap_or_else(|_| base_url.ends_with("/api"));
4508
4509        if has_sdk_api_suffix {
4510            return Err(Error::InvalidBaseUrl);
4511        }
4512
4513        Ok(Client {
4514            http: reqwest::Client::builder().timeout(self.timeout).build()?,
4515            base_url,
4516            token: self.token,
4517            control_token: self.control_token,
4518            worker_token: self.worker_token,
4519            namespace: self.namespace,
4520        })
4521    }
4522}
4523
4524#[derive(Clone, Debug)]
4525pub struct WorkflowHandle {
4526    client: Client,
4527    pub workflow_id: String,
4528    pub run_id: Option<String>,
4529    pub workflow_type: String,
4530}
4531
4532impl WorkflowHandle {
4533    /// Describe whichever run is current for this stable workflow instance.
4534    pub async fn describe(&self) -> Result<WorkflowDescription> {
4535        self.client.describe_workflow(&self.workflow_id).await
4536    }
4537
4538    /// Describe the run identity originally selected by this handle.
4539    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4540        let run_id = self.run_id.as_deref().ok_or_else(|| {
4541            Error::Codec("run_id is required for selected-run description".to_string())
4542        })?;
4543        self.client
4544            .describe_workflow_run(&self.workflow_id, run_id)
4545            .await
4546    }
4547
4548    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4549        self.client
4550            .signal_workflow(&self.workflow_id, signal_name, input)
4551            .await
4552    }
4553
4554    pub async fn append_message<T: Serialize>(
4555        &self,
4556        stream_name: &str,
4557        message_id: &str,
4558        input: T,
4559    ) -> Result<Value> {
4560        self.client
4561            .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4562            .await
4563    }
4564
4565    /// Signal only if this handle's selected run is still current.
4566    pub async fn signal_selected_run<T: Serialize>(
4567        &self,
4568        signal_name: &str,
4569        input: T,
4570    ) -> Result<Value> {
4571        let run_id = self.run_id.as_deref().ok_or_else(|| {
4572            Error::Codec("run_id is required for selected-run signaling".to_string())
4573        })?;
4574        self.client
4575            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4576            .await
4577    }
4578
4579    /// Request cooperative cancellation of whichever run is current.
4580    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4581        self.client
4582            .cancel_workflow(&self.workflow_id, options)
4583            .await
4584    }
4585
4586    /// Request cancellation only if this handle's selected run is still current.
4587    pub async fn cancel_selected_run(
4588        &self,
4589        options: WorkflowCommandOptions,
4590    ) -> Result<WorkflowCommandResult> {
4591        let run_id = self.run_id.as_deref().ok_or_else(|| {
4592            Error::Codec("run_id is required for selected-run cancellation".to_string())
4593        })?;
4594        self.client
4595            .cancel_workflow_run(&self.workflow_id, run_id, options)
4596            .await
4597    }
4598
4599    /// Forcefully terminate whichever run is current.
4600    pub async fn terminate(
4601        &self,
4602        options: WorkflowCommandOptions,
4603    ) -> Result<WorkflowCommandResult> {
4604        self.client
4605            .terminate_workflow(&self.workflow_id, options)
4606            .await
4607    }
4608
4609    /// Terminate only if this handle's selected run is still current.
4610    pub async fn terminate_selected_run(
4611        &self,
4612        options: WorkflowCommandOptions,
4613    ) -> Result<WorkflowCommandResult> {
4614        let run_id = self.run_id.as_deref().ok_or_else(|| {
4615            Error::Codec("run_id is required for selected-run termination".to_string())
4616        })?;
4617        self.client
4618            .terminate_workflow_run(&self.workflow_id, run_id, options)
4619            .await
4620    }
4621
4622    /// Execute a named, read-only query against this workflow.
4623    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4624        self.client
4625            .query_workflow(&self.workflow_id, query_name, input)
4626            .await
4627    }
4628
4629    pub async fn query_avro_value<T: Serialize>(
4630        &self,
4631        query_name: &str,
4632        input: T,
4633    ) -> Result<AvroValue> {
4634        self.client
4635            .query_workflow_avro_value(&self.workflow_id, query_name, input)
4636            .await
4637    }
4638
4639    pub async fn update<T: Serialize>(
4640        &self,
4641        update_name: &str,
4642        input: T,
4643        request_id: Option<&str>,
4644    ) -> Result<Value> {
4645        self.client
4646            .update_workflow(&self.workflow_id, update_name, input, request_id)
4647            .await
4648    }
4649
4650    pub async fn update_avro_value<T: Serialize>(
4651        &self,
4652        update_name: &str,
4653        input: T,
4654        request_id: Option<&str>,
4655    ) -> Result<AvroValue> {
4656        self.client
4657            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4658            .await
4659    }
4660
4661    /// Query only if this handle's selected run is still current.
4662    pub async fn query_selected_run<T: Serialize>(
4663        &self,
4664        query_name: &str,
4665        input: T,
4666    ) -> Result<Value> {
4667        let run_id = self
4668            .run_id
4669            .as_deref()
4670            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4671        self.client
4672            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4673            .await
4674    }
4675
4676    /// Await the final terminal outcome of the current continue-as-new chain.
4677    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4678        self.result_target(options, None).await
4679    }
4680
4681    /// Await the final result without projecting Avro bytes through JSON.
4682    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4683        self.result_avro_value_target(options, None).await
4684    }
4685
4686    /// Await the final result and decode it into a Serde application type.
4687    pub async fn result_typed<T: DeserializeOwned>(
4688        &self,
4689        options: WorkflowResultOptions,
4690    ) -> Result<T> {
4691        let result = self.result_avro_value(options).await?;
4692        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4693    }
4694
4695    /// Await only the run identity originally selected by this handle.
4696    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4697        let run_id = self.run_id.as_deref().ok_or_else(|| {
4698            Error::Codec("run_id is required for selected-run result".to_string())
4699        })?;
4700        self.result_target(options, Some(run_id)).await
4701    }
4702
4703    /// Await the selected run's result on the lossless Avro Value surface.
4704    pub async fn result_selected_run_avro_value(
4705        &self,
4706        options: WorkflowResultOptions,
4707    ) -> Result<AvroValue> {
4708        let run_id = self.run_id.as_deref().ok_or_else(|| {
4709            Error::Codec("run_id is required for selected-run result".to_string())
4710        })?;
4711        self.result_avro_value_target(options, Some(run_id)).await
4712    }
4713
4714    /// Await the selected run and decode its result into a Serde type.
4715    pub async fn result_selected_run_typed<T: DeserializeOwned>(
4716        &self,
4717        options: WorkflowResultOptions,
4718    ) -> Result<T> {
4719        let result = self.result_selected_run_avro_value(options).await?;
4720        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4721    }
4722
4723    async fn result_avro_value_target(
4724        &self,
4725        options: WorkflowResultOptions,
4726        selected_run_id: Option<&str>,
4727    ) -> Result<AvroValue> {
4728        let started = Instant::now();
4729
4730        loop {
4731            let description = match selected_run_id {
4732                Some(run_id) => {
4733                    self.client
4734                        .describe_workflow_run(&self.workflow_id, run_id)
4735                        .await?
4736                }
4737                None => self.describe().await?,
4738            };
4739            if description.is_completed() {
4740                return description.output_avro_value.ok_or_else(|| {
4741                    Error::Codec(
4742                        "missing_payload_envelope: typed workflow result requires output_envelope"
4743                            .to_string(),
4744                    )
4745                });
4746            }
4747            if description.is_terminal() {
4748                let outcome =
4749                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4750                return Err(match outcome.kind {
4751                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4752                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4753                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4754                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4755                });
4756            }
4757            if started.elapsed() >= options.timeout {
4758                return Err(Error::Timeout);
4759            }
4760            tokio::time::sleep(options.poll_interval).await;
4761        }
4762    }
4763
4764    async fn result_target(
4765        &self,
4766        options: WorkflowResultOptions,
4767        selected_run_id: Option<&str>,
4768    ) -> Result<Value> {
4769        let started = Instant::now();
4770
4771        loop {
4772            let description = match selected_run_id {
4773                Some(run_id) => {
4774                    self.client
4775                        .describe_workflow_run(&self.workflow_id, run_id)
4776                        .await?
4777                }
4778                None => self.describe().await?,
4779            };
4780            if description.is_completed() {
4781                return Ok(description.output.unwrap_or(Value::Null));
4782            }
4783
4784            if description.is_terminal() {
4785                let outcome =
4786                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4787                return Err(match outcome.kind {
4788                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4789                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4790                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4791                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4792                });
4793            }
4794
4795            if started.elapsed() >= options.timeout {
4796                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4797                    kind: WorkflowTerminalKind::TimedOut,
4798                    workflow_id: description
4799                        .workflow_id
4800                        .clone()
4801                        .unwrap_or_else(|| self.workflow_id.clone()),
4802                    run_id: description
4803                        .run_id
4804                        .clone()
4805                        .or_else(|| selected_run_id.map(str::to_string)),
4806                    reason: "result_wait_timeout".to_string(),
4807                    failure_category: Some("client_timeout".to_string()),
4808                    failure_id: None,
4809                    exception_type: None,
4810                    exception_class: None,
4811                    non_retryable: None,
4812                    message: Some(format!(
4813                        "workflow result was not terminal within {:?}",
4814                        options.timeout
4815                    )),
4816                    exception: None,
4817                    raw: description.raw_value(),
4818                }));
4819            }
4820
4821            tokio::time::sleep(options.poll_interval).await;
4822        }
4823    }
4824}
4825
4826#[derive(Clone, Copy, Debug)]
4827pub struct WorkflowResultOptions {
4828    pub poll_interval: Duration,
4829    pub timeout: Duration,
4830}
4831
4832impl Default for WorkflowResultOptions {
4833    fn default() -> Self {
4834        Self {
4835            poll_interval: Duration::from_millis(500),
4836            timeout: Duration::from_secs(30),
4837        }
4838    }
4839}
4840
4841#[derive(Clone, Debug, Deserialize)]
4842pub struct WorkflowDescription {
4843    pub workflow_id: Option<String>,
4844    pub run_id: Option<String>,
4845    pub workflow_type: Option<String>,
4846    pub status: Option<String>,
4847    #[serde(default)]
4848    pub closed_reason: Option<String>,
4849    #[serde(default)]
4850    pub error: Option<String>,
4851    #[serde(default)]
4852    pub failure: Option<Value>,
4853    #[serde(default)]
4854    pub exception: Option<Value>,
4855    #[serde(default)]
4856    pub failures: Vec<Value>,
4857    #[serde(default)]
4858    pub output: Option<Value>,
4859    #[serde(default)]
4860    pub output_envelope: Option<Value>,
4861    #[serde(skip)]
4862    pub output_avro_value: Option<AvroValue>,
4863    #[serde(flatten)]
4864    pub raw: HashMap<String, Value>,
4865}
4866
4867/// Lifecycle and backlog metadata for one run-scoped Workflow Stream.
4868#[derive(Clone, Debug, Deserialize)]
4869pub struct WorkflowStreamDescription {
4870    pub stream_name: String,
4871    pub status: String,
4872    pub last_offset: i64,
4873    pub total_items: u64,
4874    pub pending_items: u64,
4875    #[serde(default)]
4876    pub opened_at: Option<String>,
4877    #[serde(default)]
4878    pub last_appended_at: Option<String>,
4879    #[serde(default)]
4880    pub closed_at: Option<String>,
4881    #[serde(default)]
4882    pub error_reason: Option<String>,
4883    #[serde(default)]
4884    pub retention_seconds: Option<u64>,
4885    #[serde(flatten)]
4886    pub raw: HashMap<String, Value>,
4887}
4888
4889impl WorkflowStreamDescription {
4890    pub fn is_terminal(&self) -> bool {
4891        matches!(self.status.as_str(), "closed" | "errored")
4892    }
4893}
4894
4895/// One item for direct or replay-safe append.
4896#[derive(Clone, Debug, Default)]
4897pub struct WorkflowStreamAppendItem {
4898    pub payload_envelope: Option<Value>,
4899    pub payload_reference: Option<String>,
4900    pub item_type: Option<String>,
4901    pub content_type: Option<String>,
4902    pub idempotency_key: Option<String>,
4903}
4904
4905impl WorkflowStreamAppendItem {
4906    /// Encode an inline payload with the SDK's fixed Avro Value envelope.
4907    pub fn new<T: Serialize>(payload: T) -> Result<Self> {
4908        let value = AvroValue::from_serialize(&payload)?;
4909        Ok(Self {
4910            payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
4911            ..Self::default()
4912        })
4913    }
4914
4915    /// Preserve an external payload URI as an opaque service-contract reference.
4916    pub fn from_reference(reference: impl Into<String>) -> Self {
4917        Self {
4918            payload_reference: Some(reference.into()),
4919            ..Self::default()
4920        }
4921    }
4922
4923    pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
4924        self.item_type = Some(item_type.into());
4925        self
4926    }
4927
4928    pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
4929        self.content_type = Some(content_type.into());
4930        self
4931    }
4932
4933    pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
4934        self.idempotency_key = Some(idempotency_key.into());
4935        self
4936    }
4937
4938    fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
4939        let mut item = serde_json::Map::new();
4940        if let Some(payload) = &self.payload_envelope {
4941            item.insert("payload".to_string(), payload.clone());
4942            item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
4943        }
4944        if let Some(reference) = &self.payload_reference {
4945            item.insert("payload_reference".to_string(), json!(reference));
4946        }
4947        if let Some(item_type) = &self.item_type {
4948            item.insert("item_type".to_string(), json!(item_type));
4949        }
4950        if let Some(content_type) = &self.content_type {
4951            item.insert("content_type".to_string(), json!(content_type));
4952        }
4953        if let Some(key) = derived_idempotency_key
4954            .as_ref()
4955            .or(self.idempotency_key.as_ref())
4956        {
4957            item.insert("idempotency_key".to_string(), json!(key));
4958        }
4959        Value::Object(item)
4960    }
4961}
4962
4963/// One durable item at its stable zero-based offset.
4964#[derive(Clone, Debug)]
4965pub struct WorkflowStreamItem {
4966    pub offset: u64,
4967    pub payload: Option<Value>,
4968    pub payload_envelope: Option<Value>,
4969    pub payload_reference: Option<String>,
4970    pub payload_codec: Option<String>,
4971    pub idempotency_key: Option<String>,
4972    pub item_type: Option<String>,
4973    pub content_type: Option<String>,
4974    pub origin: Option<String>,
4975    pub origin_reference: Option<String>,
4976    pub emitted_at: Option<String>,
4977    pub raw: Value,
4978}
4979
4980/// One bounded at-least-once subscription page.
4981#[derive(Clone, Debug)]
4982pub struct WorkflowStreamPage {
4983    pub stream: WorkflowStreamDescription,
4984    pub items: Vec<WorkflowStreamItem>,
4985    pub next_offset: u64,
4986    pub terminal: bool,
4987}
4988
4989/// Durable acceptance and deduplication outcome for an append request.
4990#[derive(Clone, Debug)]
4991pub struct WorkflowStreamAppendResult {
4992    pub stream: WorkflowStreamDescription,
4993    pub accepted_offsets: Vec<u64>,
4994    pub accepted: u64,
4995    pub deduped: u64,
4996}
4997
4998#[derive(Deserialize)]
4999struct WorkflowStreamListResponse {
5000    #[serde(default)]
5001    streams: Vec<WorkflowStreamDescription>,
5002}
5003
5004#[derive(Deserialize)]
5005struct WorkflowStreamDescriptionResponse {
5006    stream: WorkflowStreamDescription,
5007}
5008
5009#[derive(Deserialize)]
5010struct WorkflowStreamPageResponse {
5011    stream: WorkflowStreamDescription,
5012    #[serde(default)]
5013    items: Vec<Value>,
5014    next_offset: u64,
5015    terminal: bool,
5016}
5017
5018#[derive(Deserialize)]
5019struct WorkflowStreamAppendResponse {
5020    stream: WorkflowStreamDescription,
5021    #[serde(default)]
5022    accepted_offsets: Vec<u64>,
5023    accepted: u64,
5024    deduped: u64,
5025}
5026
5027impl WorkflowDescription {
5028    pub fn is_completed(&self) -> bool {
5029        matches!(self.status.as_deref(), Some("completed" | "Completed"))
5030    }
5031
5032    pub fn is_terminal(&self) -> bool {
5033        matches!(
5034            self.status.as_deref(),
5035            Some(
5036                "completed"
5037                    | "Completed"
5038                    | "failed"
5039                    | "Failed"
5040                    | "cancelled"
5041                    | "Cancelled"
5042                    | "terminated"
5043                    | "Terminated"
5044                    | "timed_out"
5045                    | "TimedOut",
5046            )
5047        )
5048    }
5049
5050    fn decode_payloads(&mut self) -> Result<()> {
5051        if let Some(envelope) = &self.output_envelope {
5052            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5053            self.output = Some(value.clone().into_json()?);
5054            self.output_avro_value = Some(value);
5055        }
5056
5057        Ok(())
5058    }
5059
5060    fn raw_value(&self) -> Value {
5061        let mut data = self.raw.clone();
5062        data.insert(
5063            "workflow_id".to_string(),
5064            self.workflow_id
5065                .clone()
5066                .map(Value::String)
5067                .unwrap_or(Value::Null),
5068        );
5069        data.insert(
5070            "run_id".to_string(),
5071            self.run_id
5072                .clone()
5073                .map(Value::String)
5074                .unwrap_or(Value::Null),
5075        );
5076        data.insert(
5077            "workflow_type".to_string(),
5078            self.workflow_type
5079                .clone()
5080                .map(Value::String)
5081                .unwrap_or(Value::Null),
5082        );
5083        data.insert(
5084            "status".to_string(),
5085            self.status
5086                .clone()
5087                .map(Value::String)
5088                .unwrap_or(Value::Null),
5089        );
5090        data.insert(
5091            "closed_reason".to_string(),
5092            self.closed_reason
5093                .clone()
5094                .map(Value::String)
5095                .unwrap_or(Value::Null),
5096        );
5097        if let Some(failure) = &self.failure {
5098            data.insert("failure".to_string(), failure.clone());
5099        }
5100        if let Some(exception) = &self.exception {
5101            data.insert("exception".to_string(), exception.clone());
5102        }
5103        Value::Object(data.into_iter().collect())
5104    }
5105}
5106
5107fn workflow_terminal_outcome(
5108    description: &WorkflowDescription,
5109    workflow_id: &str,
5110    run_id: Option<&str>,
5111) -> WorkflowTerminalOutcome {
5112    let terminal_kind = description
5113        .closed_reason
5114        .as_deref()
5115        .or(description.status.as_deref())
5116        .unwrap_or("failed")
5117        .to_ascii_lowercase();
5118    let kind = match terminal_kind.as_str() {
5119        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5120        "terminated" => WorkflowTerminalKind::Terminated,
5121        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5122        _ => WorkflowTerminalKind::Failed,
5123    };
5124    let default_reason = match kind {
5125        WorkflowTerminalKind::Failed => "workflow_failed",
5126        WorkflowTerminalKind::Cancelled => "cancelled",
5127        WorkflowTerminalKind::Terminated => "terminated",
5128        WorkflowTerminalKind::TimedOut => "timed_out",
5129    };
5130    let failure = description
5131        .failure
5132        .as_ref()
5133        .filter(|value| value.is_object());
5134    let nested_failure = failure
5135        .and_then(|value| value.get("failures"))
5136        .and_then(Value::as_array)
5137        .and_then(|failures| failures.last())
5138        .or_else(|| description.failures.last());
5139    let exception = description
5140        .exception
5141        .clone()
5142        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5143        .or_else(|| {
5144            nested_failure
5145                .and_then(|value| value.get("exception_payload"))
5146                .cloned()
5147        });
5148    let string_field = |name: &str| {
5149        failure
5150            .and_then(|value| value.get(name))
5151            .and_then(Value::as_str)
5152            .or_else(|| {
5153                nested_failure
5154                    .and_then(|value| value.get(name))
5155                    .and_then(Value::as_str)
5156            })
5157            .map(str::to_string)
5158    };
5159    let exception_field = |name: &str| {
5160        exception
5161            .as_ref()
5162            .and_then(|value| value.get(name))
5163            .and_then(Value::as_str)
5164            .map(str::to_string)
5165    };
5166    let message = description
5167        .error
5168        .clone()
5169        .or_else(|| string_field("message"))
5170        .or_else(|| exception_field("message"));
5171    let reason = description
5172        .raw
5173        .get("reason")
5174        .and_then(Value::as_str)
5175        .map(str::to_string)
5176        .or_else(|| {
5177            failure
5178                .and_then(|value| value.get("reason"))
5179                .and_then(Value::as_str)
5180                .map(str::to_string)
5181        })
5182        .or_else(|| description.closed_reason.clone())
5183        .unwrap_or_else(|| default_reason.to_string());
5184    let failure_id = string_field("failure_id").or_else(|| {
5185        nested_failure
5186            .and_then(|value| value.get("id"))
5187            .and_then(Value::as_str)
5188            .map(str::to_string)
5189    });
5190
5191    WorkflowTerminalOutcome {
5192        kind,
5193        workflow_id: description
5194            .workflow_id
5195            .clone()
5196            .unwrap_or_else(|| workflow_id.to_string()),
5197        run_id: description
5198            .run_id
5199            .clone()
5200            .or_else(|| run_id.map(str::to_string)),
5201        reason,
5202        failure_category: string_field("failure_category")
5203            .or_else(|| Some(default_reason.to_string())),
5204        failure_id,
5205        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5206        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5207        non_retryable: failure
5208            .and_then(|value| value.get("non_retryable"))
5209            .and_then(Value::as_bool)
5210            .or_else(|| {
5211                nested_failure
5212                    .and_then(|value| value.get("non_retryable"))
5213                    .and_then(Value::as_bool)
5214            }),
5215        message,
5216        exception,
5217        raw: description.raw_value(),
5218    }
5219}
5220
5221#[derive(Clone, Debug, Deserialize)]
5222pub struct RegisterWorkerResponse {
5223    pub worker_id: String,
5224    pub registered: bool,
5225    #[serde(default)]
5226    pub heartbeat_interval_seconds: Option<u64>,
5227    #[serde(default)]
5228    pub protocol_version: Option<String>,
5229    #[serde(default)]
5230    pub server_capabilities: Option<Value>,
5231}
5232
5233/// Result of gracefully removing a worker-plane registration.
5234#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5235pub struct WorkerDeregistrationEnvelope {
5236    pub worker_id: String,
5237    pub outcome: String,
5238    pub recovered_workflow_task_count: u64,
5239}
5240
5241#[derive(Clone, Debug, Deserialize)]
5242pub struct PollWorkflowTaskResponse {
5243    #[serde(default)]
5244    pub task: Option<WorkflowTask>,
5245    #[serde(default)]
5246    pub poll_status: Option<String>,
5247    #[serde(default)]
5248    pub reason: Option<String>,
5249    #[serde(default)]
5250    pub protocol_version: Option<String>,
5251    #[serde(default)]
5252    pub server_capabilities: Option<Value>,
5253}
5254
5255impl PollWorkflowTaskResponse {
5256    /// Classify this response without parsing server display text.
5257    pub fn outcome(&self) -> WorkerPollOutcome {
5258        worker_poll_outcome(
5259            self.task.is_some(),
5260            self.poll_status.as_deref(),
5261            self.reason.as_deref(),
5262        )
5263    }
5264}
5265
5266fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5267    let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5268        return false;
5269    };
5270    let supported = capabilities
5271        .get("workflow_memo_updates")
5272        .and_then(Value::as_object)
5273        .and_then(|memo| memo.get("supported"))
5274        .and_then(Value::as_bool)
5275        == Some(true);
5276    let command_advertised = capabilities
5277        .get("supported_workflow_task_commands")
5278        .and_then(Value::as_array)
5279        .is_some_and(|commands| {
5280            commands
5281                .iter()
5282                .any(|command| command.as_str() == Some("upsert_memo"))
5283        });
5284    supported && command_advertised
5285}
5286
5287fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5288    commands
5289        .iter()
5290        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5291}
5292
5293#[derive(Clone, Debug, Deserialize)]
5294pub struct PollActivityTaskResponse {
5295    #[serde(default)]
5296    pub task: Option<ActivityTask>,
5297    #[serde(default)]
5298    pub poll_status: Option<String>,
5299    #[serde(default)]
5300    pub reason: Option<String>,
5301}
5302
5303impl PollActivityTaskResponse {
5304    /// Classify this response without parsing server display text.
5305    pub fn outcome(&self) -> WorkerPollOutcome {
5306        worker_poll_outcome(
5307            self.task.is_some(),
5308            self.poll_status.as_deref(),
5309            self.reason.as_deref(),
5310        )
5311    }
5312}
5313
5314#[derive(Clone, Debug, Deserialize)]
5315pub struct PollQueryTaskResponse {
5316    #[serde(default)]
5317    pub task: Option<QueryTask>,
5318    #[serde(default)]
5319    pub poll_status: Option<String>,
5320    #[serde(default)]
5321    pub reason: Option<String>,
5322}
5323
5324impl PollQueryTaskResponse {
5325    /// Classify this response without parsing server display text.
5326    pub fn outcome(&self) -> WorkerPollOutcome {
5327        worker_poll_outcome(
5328            self.task.is_some(),
5329            self.poll_status.as_deref(),
5330            self.reason.as_deref(),
5331        )
5332    }
5333}
5334
5335/// Stable classification for worker poll responses.
5336#[derive(Clone, Debug, PartialEq, Eq)]
5337pub enum WorkerPollOutcome {
5338    /// A task was leased and is available on the response.
5339    Task,
5340    /// No task was leased, but the worker should continue polling.
5341    Idle {
5342        poll_status: Option<String>,
5343        reason: Option<String>,
5344    },
5345    /// The server asked this worker to stop claiming new work.
5346    Stop {
5347        poll_status: Option<String>,
5348        reason: Option<String>,
5349    },
5350}
5351
5352impl WorkerPollOutcome {
5353    pub fn should_stop(&self) -> bool {
5354        matches!(self, Self::Stop { .. })
5355    }
5356}
5357
5358fn worker_poll_outcome(
5359    has_task: bool,
5360    poll_status: Option<&str>,
5361    reason: Option<&str>,
5362) -> WorkerPollOutcome {
5363    if worker_poll_is_stop(poll_status, reason) {
5364        return WorkerPollOutcome::Stop {
5365            poll_status: poll_status.map(str::to_string),
5366            reason: reason.map(str::to_string),
5367        };
5368    }
5369
5370    if has_task {
5371        WorkerPollOutcome::Task
5372    } else {
5373        WorkerPollOutcome::Idle {
5374            poll_status: poll_status.map(str::to_string),
5375            reason: reason.map(str::to_string),
5376        }
5377    }
5378}
5379
5380/// An ephemeral server-routed query task.
5381#[derive(Clone, Debug, Deserialize)]
5382pub struct QueryTask {
5383    pub query_task_id: String,
5384    #[serde(default = "default_workflow_task_attempt")]
5385    pub query_task_attempt: u64,
5386    #[serde(default)]
5387    pub lease_owner: Option<String>,
5388    #[serde(default)]
5389    pub workflow_id: Option<String>,
5390    #[serde(default)]
5391    pub run_id: Option<String>,
5392    pub workflow_type: String,
5393    pub query_name: String,
5394    #[serde(
5395        default = "missing_task_payload_codec",
5396        deserialize_with = "deserialize_task_payload_codec"
5397    )]
5398    pub payload_codec: String,
5399    #[serde(default)]
5400    pub workflow_arguments: Option<Value>,
5401    #[serde(default)]
5402    pub query_arguments: Option<Value>,
5403    #[serde(default)]
5404    pub history_events: Vec<HistoryEvent>,
5405    #[serde(default)]
5406    pub history_export: Option<Value>,
5407    #[serde(default)]
5408    pub run_status: Option<String>,
5409}
5410
5411#[derive(Clone, Debug, Deserialize)]
5412pub struct WorkflowTask {
5413    pub task_id: String,
5414    #[serde(default)]
5415    pub workflow_command_id: Option<String>,
5416    #[serde(default)]
5417    pub workflow_id: Option<String>,
5418    #[serde(default)]
5419    pub run_id: Option<String>,
5420    pub workflow_type: String,
5421    #[serde(default)]
5422    pub cancel_requested: bool,
5423    #[serde(
5424        default = "missing_task_payload_codec",
5425        deserialize_with = "deserialize_task_payload_codec"
5426    )]
5427    pub payload_codec: String,
5428    #[serde(default)]
5429    pub arguments: Option<Value>,
5430    #[serde(default)]
5431    pub history_events: Vec<HistoryEvent>,
5432    #[serde(default)]
5433    pub total_history_events: Option<u64>,
5434    #[serde(default)]
5435    pub history_size_bytes: Option<u64>,
5436    #[serde(default)]
5437    pub continue_as_new_recommended: Option<bool>,
5438    #[serde(default)]
5439    pub history_budget_pressure: Option<String>,
5440    #[serde(default)]
5441    pub next_history_page_token: Option<String>,
5442    #[serde(default = "default_workflow_task_attempt")]
5443    pub workflow_task_attempt: u64,
5444    #[serde(default)]
5445    pub workflow_signal_id: Option<String>,
5446    #[serde(default)]
5447    pub signal_name: Option<String>,
5448    #[serde(default)]
5449    pub signal_arguments: Option<Value>,
5450    #[serde(default)]
5451    pub workflow_update_id: Option<String>,
5452    #[serde(default)]
5453    pub update_name: Option<String>,
5454    #[serde(default)]
5455    pub lease_owner: Option<String>,
5456}
5457
5458impl WorkflowTask {
5459    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5460        self.history_events.extend(page.history_events);
5461
5462        if page.total_history_events.is_some() {
5463            self.total_history_events = page.total_history_events;
5464        }
5465
5466        self.next_history_page_token = page
5467            .next_history_page_token
5468            .filter(|token| !token.is_empty());
5469    }
5470}
5471
5472#[derive(Clone, Debug, Deserialize)]
5473struct WorkflowTaskHistoryPage {
5474    #[serde(default)]
5475    history_events: Vec<HistoryEvent>,
5476    #[serde(default)]
5477    total_history_events: Option<u64>,
5478    #[serde(default)]
5479    next_history_page_token: Option<String>,
5480}
5481
5482#[derive(Clone, Debug, Deserialize)]
5483pub struct ActivityTask {
5484    pub task_id: String,
5485    #[serde(default)]
5486    pub activity_attempt_id: Option<String>,
5487    #[serde(default)]
5488    pub attempt_id: Option<String>,
5489    pub activity_type: String,
5490    #[serde(
5491        default = "missing_task_payload_codec",
5492        deserialize_with = "deserialize_task_payload_codec"
5493    )]
5494    pub payload_codec: String,
5495    #[serde(default)]
5496    pub arguments: Option<Value>,
5497    #[serde(default = "default_attempt_number")]
5498    pub attempt_number: u64,
5499    #[serde(default)]
5500    pub lease_owner: Option<String>,
5501}
5502
5503#[derive(Clone, Debug, Deserialize)]
5504pub struct HistoryEvent {
5505    #[serde(alias = "type")]
5506    pub event_type: String,
5507    #[serde(default)]
5508    pub payload: Value,
5509    #[serde(flatten)]
5510    pub raw: HashMap<String, Value>,
5511}
5512
5513/// One decoded signal in the committed workflow-history snapshot.
5514#[derive(Clone, Debug, PartialEq)]
5515pub struct QuerySignal {
5516    pub id: Option<String>,
5517    pub name: String,
5518    pub arguments: Vec<Value>,
5519    avro_arguments: Vec<AvroValue>,
5520    pub workflow_sequence: Option<u64>,
5521}
5522
5523impl QuerySignal {
5524    /// Lossless fixed Avro Value arguments for this committed signal.
5525    pub fn arguments_avro_value(&self) -> &[AvroValue] {
5526        &self.avro_arguments
5527    }
5528}
5529
5530/// Immutable state supplied to a registered query handler.
5531///
5532/// This context intentionally exposes no activity, signal-wait, or command
5533/// APIs. Query handlers inspect committed history and return a value; query
5534/// completion does not append an event or advance deterministic execution.
5535#[derive(Clone, Debug)]
5536pub struct QueryContext {
5537    pub workflow_id: Option<String>,
5538    pub run_id: Option<String>,
5539    pub workflow_type: String,
5540    pub run_status: Option<String>,
5541    workflow_input: Value,
5542    workflow_input_avro_value: AvroValue,
5543    history_events: Arc<Vec<HistoryEvent>>,
5544    signal_events: Arc<Vec<QuerySignal>>,
5545}
5546
5547impl QueryContext {
5548    /// The normalized argument list used to start the workflow.
5549    pub fn workflow_input(&self) -> &Value {
5550        &self.workflow_input
5551    }
5552
5553    /// The lossless fixed Avro Value argument list used to start the workflow.
5554    pub fn workflow_input_avro_value(&self) -> &AvroValue {
5555        &self.workflow_input_avro_value
5556    }
5557
5558    /// The immutable committed history used for this query snapshot.
5559    pub fn history_events(&self) -> &[HistoryEvent] {
5560        self.history_events.as_slice()
5561    }
5562
5563    /// All decoded signals in committed workflow order.
5564    pub fn signal_events(&self) -> &[QuerySignal] {
5565        self.signal_events.as_slice()
5566    }
5567
5568    /// Decoded argument lists for each committed signal with `signal_name`.
5569    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5570        self.signal_events
5571            .iter()
5572            .filter(|signal| signal.name == signal_name)
5573            .map(|signal| signal.arguments.clone())
5574            .collect()
5575    }
5576
5577    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
5578    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5579        self.signal_events
5580            .iter()
5581            .filter(|signal| signal.name == signal_name)
5582            .map(|signal| signal.avro_arguments.clone())
5583            .collect()
5584    }
5585}
5586
5587#[derive(Clone, Debug, Deserialize)]
5588pub struct ActivityHeartbeatResponse {
5589    #[serde(default)]
5590    pub cancel_requested: bool,
5591    #[serde(default)]
5592    pub heartbeat_recorded: bool,
5593    #[serde(default)]
5594    pub can_continue: Option<bool>,
5595    #[serde(default)]
5596    pub reason: Option<String>,
5597    #[serde(default)]
5598    pub run_closed_reason: Option<String>,
5599    #[serde(default)]
5600    pub run_closed_at: Option<String>,
5601    #[serde(default)]
5602    pub lease_expires_at: Option<String>,
5603    #[serde(default)]
5604    pub last_heartbeat_at: Option<String>,
5605}
5606
5607impl ActivityHeartbeatResponse {
5608    /// Whether the activity should stop instead of attempting completion.
5609    pub fn should_stop(&self) -> bool {
5610        self.cancel_requested || self.can_continue == Some(false)
5611    }
5612}
5613
5614fn missing_task_payload_codec() -> String {
5615    MISSING_TASK_PAYLOAD_CODEC.to_string()
5616}
5617
5618fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5619where
5620    D: Deserializer<'de>,
5621{
5622    Ok(match Value::deserialize(deserializer)? {
5623        Value::String(codec) => codec,
5624        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5625        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5626    })
5627}
5628
5629fn default_workflow_task_attempt() -> u64 {
5630    1
5631}
5632
5633fn default_attempt_number() -> u64 {
5634    1
5635}
5636
5637type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5638type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5639type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5640type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5641type ReplayedWorkflowHandler =
5642    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5643type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5644type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5645type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5646type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5647type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5648type ReplayedQueryHandler = Arc<
5649    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5650        + Send
5651        + Sync,
5652>;
5653type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5654
5655struct ReplayedWorkflowInvocation {
5656    future: WorkflowFuture,
5657    snapshot: WorkflowStateSnapshot,
5658}
5659
5660#[derive(Clone)]
5661struct RegisteredWorkflow {
5662    execute: WorkflowHandler,
5663    replay: Option<ReplayedWorkflowHandler>,
5664    state_type: Option<TypeId>,
5665}
5666
5667#[derive(Debug)]
5668struct WorkflowTaskDecision {
5669    commands: Vec<Value>,
5670    message_stream_cursors: Vec<Value>,
5671    message_stream_waits: Vec<Value>,
5672}
5673
5674impl WorkflowTaskDecision {
5675    fn without_message_streams(commands: Vec<Value>) -> Self {
5676        Self {
5677            commands,
5678            message_stream_cursors: Vec::new(),
5679            message_stream_waits: Vec::new(),
5680        }
5681    }
5682}
5683
5684#[derive(Clone)]
5685enum RegisteredQuery {
5686    Snapshot(QueryHandler),
5687    Replayed {
5688        state_type: TypeId,
5689        handler: ReplayedQueryHandler,
5690    },
5691}
5692
5693#[derive(Clone, Debug)]
5694pub struct WorkerHeartbeatObservation {
5695    pub worker_id: String,
5696    pub task_queue: String,
5697    pub acknowledged_at_unix_millis: u64,
5698    pub acknowledgement: Value,
5699}
5700
5701/// Bounded retry policy for worker poll acquisition and worker heartbeats.
5702///
5703/// Expected empty long polls and explicit Server capacity backpressure are
5704/// normal worker states. Capacity backpressure honors the advertised retry
5705/// delay without consuming this retry budget. Other transport failures,
5706/// retryable HTTP 408/429 responses, and server errors are retried with capped
5707/// exponential backoff. Authentication, protocol, codec, and handler failures
5708/// are never retried by the worker.
5709#[derive(Clone, Copy, Debug)]
5710pub struct WorkerRetryPolicy {
5711    /// Number of retries after the initial request fails.
5712    pub max_retries: usize,
5713    /// Delay before the first retry.
5714    pub initial_backoff: Duration,
5715    /// Maximum delay between retries.
5716    pub max_backoff: Duration,
5717}
5718
5719impl Default for WorkerRetryPolicy {
5720    fn default() -> Self {
5721        Self {
5722            max_retries: 5,
5723            initial_backoff: Duration::from_millis(100),
5724            max_backoff: Duration::from_secs(5),
5725        }
5726    }
5727}
5728
5729#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5730enum ManagedPollOutcome {
5731    Idle,
5732    Handled,
5733    Stop,
5734}
5735
5736#[derive(Clone)]
5737pub struct Worker {
5738    client: Client,
5739    worker_id: String,
5740    task_queue: String,
5741    workflows: HashMap<String, RegisteredWorkflow>,
5742    activities: HashMap<String, ActivityHandler>,
5743    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5744    updates: HashMap<String, HashMap<String, UpdateHandler>>,
5745    max_concurrent_workflow_tasks: usize,
5746    max_concurrent_activity_tasks: usize,
5747    poll_timeout: Duration,
5748    heartbeat_interval: Duration,
5749    retry_policy: WorkerRetryPolicy,
5750    heartbeat_observer: Option<WorkerHeartbeatObserver>,
5751}
5752
5753impl Worker {
5754    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5755        Self {
5756            client,
5757            worker_id: default_worker_id(),
5758            task_queue: task_queue.into(),
5759            workflows: HashMap::new(),
5760            activities: HashMap::new(),
5761            queries: HashMap::new(),
5762            updates: HashMap::new(),
5763            max_concurrent_workflow_tasks: 10,
5764            max_concurrent_activity_tasks: 10,
5765            poll_timeout: Duration::from_secs(30),
5766            heartbeat_interval: Duration::from_secs(60),
5767            retry_policy: WorkerRetryPolicy::default(),
5768            heartbeat_observer: None,
5769        }
5770    }
5771
5772    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5773        self.worker_id = worker_id.into();
5774        self
5775    }
5776
5777    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5778        self.poll_timeout = timeout;
5779        self
5780    }
5781
5782    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5783        self.heartbeat_interval = interval;
5784        self
5785    }
5786
5787    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
5788    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5789        self.retry_policy = policy;
5790        self
5791    }
5792
5793    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5794    where
5795        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5796    {
5797        self.heartbeat_observer = Some(Arc::new(observer));
5798        self
5799    }
5800
5801    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5802        self.max_concurrent_workflow_tasks = count.max(1);
5803        self
5804    }
5805
5806    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5807        self.max_concurrent_activity_tasks = count.max(1);
5808        self
5809    }
5810
5811    /// Register a workflow handler.
5812    ///
5813    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
5814    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
5815    /// while acquiring or decoding a worker task remain worker-operation
5816    /// failures and do not get converted into workflow outcomes.
5817    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5818    where
5819        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5820        Fut: Future<Output = Result<Value>> + Send + 'static,
5821    {
5822        let handler = Arc::new(handler);
5823        self.workflows.insert(
5824            workflow_type.into(),
5825            RegisteredWorkflow {
5826                execute: Arc::new(move |ctx, input| {
5827                    let handler = Arc::clone(&handler);
5828                    Box::pin(async move {
5829                        let result = handler(ctx, input.into_json()?).await?;
5830                        AvroValue::from_serialize(&result)
5831                    })
5832                }),
5833                replay: None,
5834                state_type: None,
5835            },
5836        );
5837    }
5838
5839    /// Register a workflow with one Serde request value and a Serde result.
5840    ///
5841    /// This is an ergonomic adapter over the same fixed Avro Value protocol as
5842    /// [`Worker::register_workflow_avro_value`]. It does not create or publish a
5843    /// workflow-specific schema. A task must contain zero arguments for a unit
5844    /// request or exactly one argument for every other request type.
5845    ///
5846    /// See the runnable
5847    /// [`hello_world` example](https://github.com/durable-workflow/sdk-rust/blob/main/examples/hello_world.rs)
5848    /// for typed workflow and activity contracts with retry and timeout policy.
5849    pub fn register_typed_workflow<I, O, F, Fut>(
5850        &mut self,
5851        workflow_type: impl Into<String>,
5852        handler: F,
5853    ) where
5854        I: DeserializeOwned + Send + 'static,
5855        O: Serialize + Send + 'static,
5856        F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
5857        Fut: Future<Output = Result<O>> + Send + 'static,
5858    {
5859        let workflow_type = workflow_type.into();
5860        let handler_name = workflow_type.clone();
5861        let handler = Arc::new(handler);
5862        self.workflows.insert(
5863            workflow_type,
5864            RegisteredWorkflow {
5865                execute: Arc::new(move |ctx, input| {
5866                    let handler = Arc::clone(&handler);
5867                    let handler_name = handler_name.clone();
5868                    Box::pin(async move {
5869                        let input =
5870                            decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5871                        let result = handler(ctx, input).await?;
5872                        encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5873                    })
5874                }),
5875                replay: None,
5876                state_type: None,
5877            },
5878        );
5879    }
5880
5881    /// Register a workflow on the lossless fixed Avro Value surface.
5882    pub fn register_workflow_avro_value<F, Fut>(
5883        &mut self,
5884        workflow_type: impl Into<String>,
5885        handler: F,
5886    ) where
5887        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
5888        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5889    {
5890        self.workflows.insert(
5891            workflow_type.into(),
5892            RegisteredWorkflow {
5893                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
5894                replay: None,
5895                state_type: None,
5896            },
5897        );
5898    }
5899
5900    /// Register a workflow whose typed instance state can be reconstructed for queries.
5901    ///
5902    /// `state_factory` creates a fresh instance for every normal workflow task and
5903    /// query replay. The workflow handler is the single source of truth for state
5904    /// transitions: it updates [`WorkflowInstance`] after activities and signals
5905    /// resolve. Query replay runs this same handler over committed history and
5906    /// discards any commands it would emit.
5907    pub fn register_replayed_workflow<S, Factory, F, Fut>(
5908        &mut self,
5909        workflow_type: impl Into<String>,
5910        state_factory: Factory,
5911        handler: F,
5912    ) where
5913        S: Clone + Send + Sync + 'static,
5914        Factory: Fn() -> S + Send + Sync + 'static,
5915        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5916        Fut: Future<Output = Result<Value>> + Send + 'static,
5917    {
5918        let state_factory = Arc::new(state_factory);
5919        let handler = Arc::new(handler);
5920
5921        let execute_factory = Arc::clone(&state_factory);
5922        let execute_handler = Arc::clone(&handler);
5923        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5924            let state = WorkflowInstance::new(execute_factory());
5925            let handler = Arc::clone(&execute_handler);
5926            Box::pin(async move {
5927                let result = handler(ctx, input.into_json()?, state).await?;
5928                AvroValue::from_serialize(&result)
5929            }) as WorkflowFuture
5930        });
5931
5932        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5933            let state = WorkflowInstance::new(state_factory());
5934            let snapshot_state = state.clone();
5935            let snapshot: WorkflowStateSnapshot =
5936                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5937            let replay_handler = Arc::clone(&handler);
5938            let future = async move {
5939                let result = replay_handler(ctx, input.into_json()?, state).await?;
5940                AvroValue::from_serialize(&result)
5941            };
5942            ReplayedWorkflowInvocation {
5943                future: Box::pin(future),
5944                snapshot,
5945            }
5946        });
5947
5948        self.workflows.insert(
5949            workflow_type.into(),
5950            RegisteredWorkflow {
5951                execute,
5952                replay: Some(replay),
5953                state_type: Some(TypeId::of::<S>()),
5954            },
5955        );
5956    }
5957
5958    /// Register a replayable workflow with one Serde request value and result.
5959    ///
5960    /// Normal task execution and instance-state query replay both decode and
5961    /// encode through the fixed Avro Value codec. The state factory and handler
5962    /// otherwise follow [`Worker::register_replayed_workflow`].
5963    pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
5964        &mut self,
5965        workflow_type: impl Into<String>,
5966        state_factory: Factory,
5967        handler: F,
5968    ) where
5969        I: DeserializeOwned + Send + 'static,
5970        O: Serialize + Send + 'static,
5971        S: Clone + Send + Sync + 'static,
5972        Factory: Fn() -> S + Send + Sync + 'static,
5973        F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5974        Fut: Future<Output = Result<O>> + Send + 'static,
5975    {
5976        let workflow_type = workflow_type.into();
5977        let state_factory = Arc::new(state_factory);
5978        let handler = Arc::new(handler);
5979
5980        let execute_name = workflow_type.clone();
5981        let execute_factory = Arc::clone(&state_factory);
5982        let execute_handler = Arc::clone(&handler);
5983        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5984            let state = WorkflowInstance::new(execute_factory());
5985            let handler = Arc::clone(&execute_handler);
5986            let handler_name = execute_name.clone();
5987            Box::pin(async move {
5988                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5989                let result = handler(ctx, input, state).await?;
5990                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5991            }) as WorkflowFuture
5992        });
5993
5994        let replay_name = workflow_type.clone();
5995        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5996            let state = WorkflowInstance::new(state_factory());
5997            let snapshot_state = state.clone();
5998            let snapshot: WorkflowStateSnapshot =
5999                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6000            let handler = Arc::clone(&handler);
6001            let handler_name = replay_name.clone();
6002            let future = async move {
6003                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6004                let result = handler(ctx, input, state).await?;
6005                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6006            };
6007            ReplayedWorkflowInvocation {
6008                future: Box::pin(future),
6009                snapshot,
6010            }
6011        });
6012
6013        self.workflows.insert(
6014            workflow_type,
6015            RegisteredWorkflow {
6016                execute,
6017                replay: Some(replay),
6018                state_type: Some(TypeId::of::<S>()),
6019            },
6020        );
6021    }
6022
6023    /// Register a replayable workflow on the lossless fixed Avro Value surface.
6024    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6025        &mut self,
6026        workflow_type: impl Into<String>,
6027        state_factory: Factory,
6028        handler: F,
6029    ) where
6030        S: Clone + Send + Sync + 'static,
6031        Factory: Fn() -> S + Send + Sync + 'static,
6032        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6033        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6034    {
6035        let state_factory = Arc::new(state_factory);
6036        let handler = Arc::new(handler);
6037
6038        let execute_factory = Arc::clone(&state_factory);
6039        let execute_handler = Arc::clone(&handler);
6040        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6041            let state = WorkflowInstance::new(execute_factory());
6042            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6043        });
6044
6045        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6046            let state = WorkflowInstance::new(state_factory());
6047            let snapshot_state = state.clone();
6048            let snapshot: WorkflowStateSnapshot =
6049                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6050            ReplayedWorkflowInvocation {
6051                future: Box::pin(handler(ctx, input, state)),
6052                snapshot,
6053            }
6054        });
6055
6056        self.workflows.insert(
6057            workflow_type.into(),
6058            RegisteredWorkflow {
6059                execute,
6060                replay: Some(replay),
6061                state_type: Some(TypeId::of::<S>()),
6062            },
6063        );
6064    }
6065
6066    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6067    where
6068        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6069        Fut: Future<Output = Result<Value>> + Send + 'static,
6070    {
6071        let handler = Arc::new(handler);
6072        self.activities.insert(
6073            activity_type.into(),
6074            Arc::new(move |ctx, args| {
6075                let handler = Arc::clone(&handler);
6076                Box::pin(async move {
6077                    let result = handler(ctx, args.into_json()?).await?;
6078                    AvroValue::from_serialize(&result)
6079                })
6080            }),
6081        );
6082    }
6083
6084    /// Register an activity with one Serde request value and a Serde result.
6085    ///
6086    /// Inputs and results use the platform's fixed Avro Value schema. Shape
6087    /// mismatches and unsupported Serde values return [`Error::HandlerType`]
6088    /// with the activity name and Rust type.
6089    pub fn register_typed_activity<I, O, F, Fut>(
6090        &mut self,
6091        activity_type: impl Into<String>,
6092        handler: F,
6093    ) where
6094        I: DeserializeOwned + Send + 'static,
6095        O: Serialize + Send + 'static,
6096        F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6097        Fut: Future<Output = Result<O>> + Send + 'static,
6098    {
6099        let activity_type = activity_type.into();
6100        let handler_name = activity_type.clone();
6101        let handler = Arc::new(handler);
6102        self.activities.insert(
6103            activity_type,
6104            Arc::new(move |ctx, input| {
6105                let handler = Arc::clone(&handler);
6106                let handler_name = handler_name.clone();
6107                Box::pin(async move {
6108                    let input =
6109                        decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6110                    let result = handler(ctx, input).await?;
6111                    encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6112                })
6113            }),
6114        );
6115    }
6116
6117    /// Register an activity on the lossless fixed Avro Value surface.
6118    pub fn register_activity_avro_value<F, Fut>(
6119        &mut self,
6120        activity_type: impl Into<String>,
6121        handler: F,
6122    ) where
6123        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6124        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6125    {
6126        self.activities.insert(
6127            activity_type.into(),
6128            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6129        );
6130    }
6131
6132    /// Register a named, read-only query handler for a workflow type.
6133    ///
6134    /// The workflow type must also be registered with [`Worker::register_workflow`]
6135    /// before the worker runs. The handler receives only an immutable committed
6136    /// state snapshot and normalized query arguments.
6137    pub fn register_query<F, Fut>(
6138        &mut self,
6139        workflow_type: impl Into<String>,
6140        query_name: impl Into<String>,
6141        handler: F,
6142    ) where
6143        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6144        Fut: Future<Output = Result<Value>> + Send + 'static,
6145    {
6146        let handler = Arc::new(handler);
6147        self.queries
6148            .entry(workflow_type.into())
6149            .or_default()
6150            .insert(
6151                query_name.into(),
6152                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6153                    let handler = Arc::clone(&handler);
6154                    Box::pin(async move {
6155                        let result = handler(ctx, args.into_json()?).await?;
6156                        AvroValue::from_serialize(&result)
6157                    })
6158                })),
6159            );
6160    }
6161
6162    /// Register a query handler on the lossless fixed Avro Value surface.
6163    pub fn register_query_avro_value<F, Fut>(
6164        &mut self,
6165        workflow_type: impl Into<String>,
6166        query_name: impl Into<String>,
6167        handler: F,
6168    ) where
6169        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6170        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6171    {
6172        self.queries
6173            .entry(workflow_type.into())
6174            .or_default()
6175            .insert(
6176                query_name.into(),
6177                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6178            );
6179    }
6180
6181    /// Register a named query against deterministically replayed instance state.
6182    ///
6183    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
6184    /// same state type `S`. The handler receives an immutable, detached state
6185    /// clone, so successful and failed queries cannot affect workflow execution
6186    /// or the state reconstructed by a later query.
6187    pub fn register_replayed_query<S, F, Fut>(
6188        &mut self,
6189        workflow_type: impl Into<String>,
6190        query_name: impl Into<String>,
6191        handler: F,
6192    ) where
6193        S: Clone + Send + Sync + 'static,
6194        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6195        Fut: Future<Output = Result<Value>> + Send + 'static,
6196    {
6197        let handler = Arc::new(handler);
6198        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6199            let state = state.downcast::<S>().map_err(|_| {
6200                "registered query state type does not match the replayed workflow state".to_string()
6201            })?;
6202            let handler = Arc::clone(&handler);
6203            Ok(Box::pin(async move {
6204                let result = handler(ctx, state, args.into_json()?).await?;
6205                AvroValue::from_serialize(&result)
6206            }))
6207        });
6208
6209        self.queries
6210            .entry(workflow_type.into())
6211            .or_default()
6212            .insert(
6213                query_name.into(),
6214                RegisteredQuery::Replayed {
6215                    state_type: TypeId::of::<S>(),
6216                    handler: erased_handler,
6217                },
6218            );
6219    }
6220
6221    /// Register a replayed-state query on the lossless fixed Avro Value surface.
6222    pub fn register_replayed_query_avro_value<S, F, Fut>(
6223        &mut self,
6224        workflow_type: impl Into<String>,
6225        query_name: impl Into<String>,
6226        handler: F,
6227    ) where
6228        S: Clone + Send + Sync + 'static,
6229        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6230        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6231    {
6232        let handler = Arc::new(handler);
6233        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6234            let state = state.downcast::<S>().map_err(|_| {
6235                "registered query state type does not match the replayed workflow state".to_string()
6236            })?;
6237            Ok(Box::pin(handler(ctx, state, args)))
6238        });
6239
6240        self.queries
6241            .entry(workflow_type.into())
6242            .or_default()
6243            .insert(
6244                query_name.into(),
6245                RegisteredQuery::Replayed {
6246                    state_type: TypeId::of::<S>(),
6247                    handler: erased_handler,
6248                },
6249            );
6250    }
6251
6252    /// Register a synchronous workflow update handler.
6253    pub fn register_update<F, Fut>(
6254        &mut self,
6255        workflow_type: impl Into<String>,
6256        update_name: impl Into<String>,
6257        handler: F,
6258    ) where
6259        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6260        Fut: Future<Output = Result<Value>> + Send + 'static,
6261    {
6262        let handler = Arc::new(handler);
6263        self.updates
6264            .entry(workflow_type.into())
6265            .or_default()
6266            .insert(
6267                update_name.into(),
6268                Arc::new(move |ctx, args| {
6269                    let handler = Arc::clone(&handler);
6270                    Box::pin(async move {
6271                        let result = handler(ctx, args.into_json()?).await?;
6272                        AvroValue::from_serialize(&result)
6273                    })
6274                }),
6275            );
6276    }
6277
6278    /// Register an update handler on the lossless fixed Avro Value surface.
6279    pub fn register_update_avro_value<F, Fut>(
6280        &mut self,
6281        workflow_type: impl Into<String>,
6282        update_name: impl Into<String>,
6283        handler: F,
6284    ) where
6285        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6286        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6287    {
6288        self.updates
6289            .entry(workflow_type.into())
6290            .or_default()
6291            .insert(
6292                update_name.into(),
6293                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6294            );
6295    }
6296
6297    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6298        let mut command_contracts = serde_json::Map::new();
6299        for workflow_type in self.workflows.keys() {
6300            let mut queries = self
6301                .queries
6302                .get(workflow_type)
6303                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6304                .unwrap_or_default();
6305            queries.sort();
6306            let mut updates = self
6307                .updates
6308                .get(workflow_type)
6309                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6310                .unwrap_or_default();
6311            updates.sort();
6312            command_contracts.insert(
6313                workflow_type.clone(),
6314                json!({
6315                    "queries": queries,
6316                    "query_contracts": [],
6317                    "signals": [],
6318                    "signal_contracts": [],
6319                    "updates": updates,
6320                    "update_contracts": [],
6321                    "update_validators": [],
6322                }),
6323            );
6324        }
6325
6326        self.client
6327            .register_worker_with_command_contracts(
6328                &self.worker_id,
6329                &self.task_queue,
6330                self.workflows.keys().cloned().collect(),
6331                self.activities.keys().cloned().collect(),
6332                self.max_concurrent_workflow_tasks,
6333                self.max_concurrent_activity_tasks,
6334                [
6335                    Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6336                    Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6337                    Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6338                    Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6339                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6340                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6341                    worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6342                        .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6343                ]
6344                .into_iter()
6345                .flatten()
6346                .collect(),
6347                Value::Object(command_contracts),
6348            )
6349            .await
6350    }
6351
6352    /// Run until shutdown or a terminal worker error occurs.
6353    ///
6354    /// Empty long-poll expirations do not stop the worker. Retryable poll and
6355    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
6356    /// authentication, protocol, and other non-retryable failures are returned.
6357    pub async fn run(&self) -> Result<()> {
6358        self.run_until(std::future::pending::<()>()).await
6359    }
6360
6361    /// Run until `shutdown` resolves or a terminal worker error occurs.
6362    ///
6363    /// This has the same liveness and terminal-error contract as [`Worker::run`].
6364    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6365    where
6366        F: Future<Output = ()>,
6367    {
6368        let registration = self.register().await?;
6369        if !registration.registered {
6370            return Err(Error::WorkerLoop(format!(
6371                "worker registration for {:?} was not accepted",
6372                self.worker_id
6373            )));
6374        }
6375        let registered_worker_id = registration.worker_id.clone();
6376        let primary = self.run_registered_until(shutdown, registration).await;
6377        let deregistration = self
6378            .client
6379            .deregister_worker_registration(&registered_worker_id)
6380            .await;
6381
6382        match (primary, deregistration) {
6383            (Ok(()), Ok(_)) => Ok(()),
6384            (Ok(()), Err(deregistration)) => Err(deregistration),
6385            (Err(primary), Ok(_)) => Err(primary),
6386            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6387                primary: Box::new(primary),
6388                deregistration: Box::new(deregistration),
6389            }),
6390        }
6391    }
6392
6393    async fn run_registered_until<F>(
6394        &self,
6395        shutdown: F,
6396        registration: RegisterWorkerResponse,
6397    ) -> Result<()>
6398    where
6399        F: Future<Output = ()>,
6400    {
6401        let heartbeat_interval = Duration::from_secs(
6402            registration
6403                .heartbeat_interval_seconds
6404                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6405        );
6406        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
6407        // from the completion of the preceding attempt, including its bounded
6408        // retries. A fixed-epoch interval can leave an already-due tick queued
6409        // while an acknowledgement is slow, producing a catch-up heartbeat as
6410        // soon as that request completes.
6411        let heartbeat = tokio::time::sleep(Duration::ZERO);
6412        tokio::pin!(heartbeat);
6413        tokio::pin!(shutdown);
6414        let stop = Arc::new(AtomicBool::new(false));
6415        // Poll responses may already have leased server-side work by the time
6416        // they become ready, so each poller owns its responses through
6417        // completion or failure instead of racing raw polls in this select.
6418        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6419            let worker = self.clone();
6420            let stop = Arc::clone(&stop);
6421            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6422        });
6423        let mut activity_poller = (!self.activities.is_empty()).then(|| {
6424            let worker = self.clone();
6425            let stop = Arc::clone(&stop);
6426            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6427        });
6428        let mut query_poller = (!self.queries.is_empty()).then(|| {
6429            let worker = self.clone();
6430            let stop = Arc::clone(&stop);
6431            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6432        });
6433
6434        loop {
6435            tokio::select! {
6436                _ = &mut shutdown => {
6437                    stop.store(true, Ordering::SeqCst);
6438                    break;
6439                }
6440                _ = &mut heartbeat => {
6441                    let result = self.retry_worker_operation(|| {
6442                        self.client.heartbeat_worker(
6443                            &self.worker_id,
6444                            self.max_concurrent_workflow_tasks,
6445                            self.max_concurrent_activity_tasks,
6446                        )
6447                    }).await;
6448                    heartbeat
6449                        .as_mut()
6450                        .reset(tokio::time::Instant::now() + heartbeat_interval);
6451                    match result {
6452                        Ok(acknowledgement) => {
6453                            if let Some(observer) = &self.heartbeat_observer {
6454                                observer(&WorkerHeartbeatObservation {
6455                                    worker_id: self.worker_id.clone(),
6456                                    task_queue: self.task_queue.clone(),
6457                                    acknowledged_at_unix_millis: SystemTime::now()
6458                                        .duration_since(UNIX_EPOCH)
6459                                        .unwrap_or_default()
6460                                        .as_millis()
6461                                        .min(u64::MAX as u128)
6462                                        as u64,
6463                                    acknowledgement,
6464                                });
6465                            }
6466                        }
6467                        Err(error) => {
6468                            stop.store(true, Ordering::SeqCst);
6469                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6470                            return Err(error);
6471                        }
6472                    }
6473                }
6474                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6475                    workflow_poller = None;
6476                    let stopped_by_server = stop.load(Ordering::SeqCst);
6477                    stop.store(true, Ordering::SeqCst);
6478                    let poller_result = optional_poller_result("workflow", result);
6479                    let join_result =
6480                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6481                    poller_result?;
6482                    join_result?;
6483                    if stopped_by_server {
6484                        return Ok(());
6485                    }
6486                    return Err(Error::WorkerLoop(
6487                        "workflow poller stopped unexpectedly".to_string(),
6488                    ));
6489                }
6490                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6491                    activity_poller = None;
6492                    let stopped_by_server = stop.load(Ordering::SeqCst);
6493                    stop.store(true, Ordering::SeqCst);
6494                    let poller_result = optional_poller_result("activity", result);
6495                    let join_result =
6496                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6497                    poller_result?;
6498                    join_result?;
6499                    if stopped_by_server {
6500                        return Ok(());
6501                    }
6502                    return Err(Error::WorkerLoop(
6503                        "activity poller stopped unexpectedly".to_string(),
6504                    ));
6505                }
6506                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6507                    query_poller = None;
6508                    let stopped_by_server = stop.load(Ordering::SeqCst);
6509                    stop.store(true, Ordering::SeqCst);
6510                    let poller_result = optional_poller_result("query", result);
6511                    let join_result =
6512                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6513                    poller_result?;
6514                    join_result?;
6515                    if stopped_by_server {
6516                        return Ok(());
6517                    }
6518                    return Err(Error::WorkerLoop(
6519                        "query poller stopped unexpectedly".to_string(),
6520                    ));
6521                }
6522            }
6523        }
6524
6525        join_pollers(
6526            workflow_poller.take(),
6527            activity_poller.take(),
6528            query_poller.take(),
6529        )
6530        .await
6531    }
6532
6533    /// Poll and settle at most one task from each enabled task family.
6534    ///
6535    /// A workflow may reach its server-enforced run deadline while this worker
6536    /// holds a task. When the completion endpoint authoritatively rejects that
6537    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
6538    /// terminal `run_status=failed`, the workflow tick is considered settled:
6539    /// the late command was not recorded and cannot replace the terminal run.
6540    /// Every other completion rejection remains an error. This worker-level
6541    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
6542    /// only bounds how long a client waits for a result.
6543    ///
6544    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
6545    /// the original [`Error::Http`] status and response body.
6546    pub async fn run_once(&self) -> Result<usize> {
6547        let mut handled = 0;
6548        match self.poll_workflow_once().await? {
6549            ManagedPollOutcome::Handled => handled += 1,
6550            ManagedPollOutcome::Stop => return Ok(handled),
6551            ManagedPollOutcome::Idle => {}
6552        }
6553        match self.poll_activity_once().await? {
6554            ManagedPollOutcome::Handled => handled += 1,
6555            ManagedPollOutcome::Stop => return Ok(handled),
6556            ManagedPollOutcome::Idle => {}
6557        }
6558        if !self.queries.is_empty() {
6559            match self.poll_query_once().await? {
6560                ManagedPollOutcome::Handled => handled += 1,
6561                ManagedPollOutcome::Stop => return Ok(handled),
6562                ManagedPollOutcome::Idle => {}
6563            }
6564        }
6565        Ok(handled)
6566    }
6567
6568    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6569        let poll_request_id = unique_request_id("rust-workflow-poll");
6570        let response = self
6571            .retry_worker_operation(|| {
6572                self.client.poll_workflow_task_response_with_request_id(
6573                    &self.worker_id,
6574                    &self.task_queue,
6575                    self.poll_timeout,
6576                    &poll_request_id,
6577                    0,
6578                )
6579            })
6580            .await;
6581        let Some(response) = self.settle_worker_poll_response(response).await? else {
6582            return Ok(ManagedPollOutcome::Idle);
6583        };
6584        if response.outcome().should_stop() {
6585            return Ok(ManagedPollOutcome::Stop);
6586        }
6587        let memo_updates_supported =
6588            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6589        let Some(task) = response.task else {
6590            return Ok(ManagedPollOutcome::Idle);
6591        };
6592
6593        let task_id = task.task_id.clone();
6594        let attempt = task.workflow_task_attempt;
6595        let run_id = task.run_id.clone();
6596        let lease_owner = task
6597            .lease_owner
6598            .clone()
6599            .unwrap_or_else(|| self.worker_id.clone());
6600
6601        match self.execute_workflow_task_decision(task) {
6602            Ok(decision)
6603                if commands_use_workflow_memo_updates(&decision.commands)
6604                    && !memo_updates_supported =>
6605            {
6606                self.client
6607                    .fail_workflow_task(
6608                        &task_id,
6609                        &lease_owner,
6610                        attempt,
6611                        Error::WorkflowMemoUpdatesUnavailable.to_string(),
6612                    )
6613                    .await?;
6614            }
6615            Ok(decision) if decision.commands.is_empty() => {
6616                // A replay can consume a recorded pending durable command
6617                // without producing a new command. The standalone protocol
6618                // acknowledges that state through the typed waiting outcome;
6619                // an empty completion is rejected by servers that require at
6620                // least one executable command.
6621                self.client
6622                    .fail_workflow_task_with_type(
6623                        &task_id,
6624                        &lease_owner,
6625                        attempt,
6626                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6627                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6628                    )
6629                    .await?;
6630            }
6631            Ok(decision) => {
6632                let completion = self
6633                    .client
6634                    .complete_workflow_task_with_message_streams(
6635                        &task_id,
6636                        &lease_owner,
6637                        attempt,
6638                        decision.commands,
6639                        decision.message_stream_cursors,
6640                        decision.message_stream_waits,
6641                    )
6642                    .await;
6643                if let Err(error) = completion {
6644                    if !workflow_task_completion_is_terminal_timeout(
6645                        &error,
6646                        &task_id,
6647                        attempt,
6648                        run_id.as_deref(),
6649                    ) {
6650                        return Err(error);
6651                    }
6652                }
6653            }
6654            Err(error) => {
6655                self.client
6656                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6657                    .await?;
6658            }
6659        }
6660
6661        Ok(ManagedPollOutcome::Handled)
6662    }
6663
6664    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6665        while !stop.load(Ordering::SeqCst) {
6666            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6667                stop.store(true, Ordering::SeqCst);
6668                break;
6669            }
6670        }
6671
6672        Ok(())
6673    }
6674
6675    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6676        let poll_request_id = unique_request_id("rust-activity-poll");
6677        let response = self
6678            .retry_worker_operation(|| {
6679                self.client.poll_activity_task_response_with_request_id(
6680                    &self.worker_id,
6681                    &self.task_queue,
6682                    self.poll_timeout,
6683                    &poll_request_id,
6684                    0,
6685                )
6686            })
6687            .await;
6688        let Some(response) = self.settle_worker_poll_response(response).await? else {
6689            return Ok(ManagedPollOutcome::Idle);
6690        };
6691        if response.outcome().should_stop() {
6692            return Ok(ManagedPollOutcome::Stop);
6693        }
6694        let Some(task) = response.task else {
6695            return Ok(ManagedPollOutcome::Idle);
6696        };
6697
6698        let task_id = task.task_id.clone();
6699        let attempt_id = task
6700            .activity_attempt_id
6701            .clone()
6702            .or(task.attempt_id.clone())
6703            .unwrap_or_default();
6704        let lease_owner = task
6705            .lease_owner
6706            .clone()
6707            .unwrap_or_else(|| self.worker_id.clone());
6708        let codec = task.payload_codec.clone();
6709        let result = self.execute_activity_task(task).await;
6710        match result {
6711            Ok(value) => {
6712                let completion = self
6713                    .client
6714                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6715                    .await;
6716                if let Err(error) = completion {
6717                    if !activity_task_rejection_is_final(&error) {
6718                        return Err(error);
6719                    }
6720                }
6721            }
6722            Err(error) => {
6723                let failure = self
6724                    .client
6725                    .fail_activity_task(
6726                        &task_id,
6727                        &attempt_id,
6728                        &lease_owner,
6729                        error.to_string(),
6730                        false,
6731                    )
6732                    .await;
6733                if let Err(error) = failure {
6734                    if !activity_task_rejection_is_final(&error) {
6735                        return Err(error);
6736                    }
6737                }
6738            }
6739        }
6740
6741        Ok(ManagedPollOutcome::Handled)
6742    }
6743
6744    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6745        while !stop.load(Ordering::SeqCst) {
6746            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6747                stop.store(true, Ordering::SeqCst);
6748                break;
6749            }
6750        }
6751
6752        Ok(())
6753    }
6754
6755    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6756        let poll_request_id = unique_request_id("rust-query-poll");
6757        let response = self
6758            .retry_worker_operation(|| {
6759                self.client.poll_query_task_response_with_request_id(
6760                    &self.worker_id,
6761                    &self.task_queue,
6762                    self.poll_timeout,
6763                    &poll_request_id,
6764                    0,
6765                )
6766            })
6767            .await;
6768        let Some(response) = self.settle_worker_poll_response(response).await? else {
6769            return Ok(ManagedPollOutcome::Idle);
6770        };
6771        if response.outcome().should_stop() {
6772            return Ok(ManagedPollOutcome::Stop);
6773        }
6774        let Some(task) = response.task else {
6775            return Ok(ManagedPollOutcome::Idle);
6776        };
6777
6778        let query_task_id = task.query_task_id.clone();
6779        let attempt = task.query_task_attempt;
6780        let lease_owner = task
6781            .lease_owner
6782            .clone()
6783            .unwrap_or_else(|| self.worker_id.clone());
6784        let codec = task.payload_codec.clone();
6785
6786        match self.execute_query_task(task).await {
6787            Ok(value) => {
6788                let result_envelope = match encode_typed_envelope(&value, &codec) {
6789                    Ok(result_envelope) => result_envelope,
6790                    Err(error) => {
6791                        let failure = self
6792                            .client
6793                            .fail_query_task(
6794                                &query_task_id,
6795                                &lease_owner,
6796                                attempt,
6797                                error.to_string(),
6798                                "query_result_encode_failed",
6799                                "QueryResultEncodeFailed",
6800                            )
6801                            .await;
6802                        if let Err(error) = failure {
6803                            if !query_task_rejection_is_final(&error) {
6804                                return Err(error);
6805                            }
6806                        }
6807                        return Ok(ManagedPollOutcome::Handled);
6808                    }
6809                };
6810
6811                if let Err(error) = self
6812                    .client
6813                    .complete_query_task_with_envelope(
6814                        &query_task_id,
6815                        &lease_owner,
6816                        attempt,
6817                        value.clone().into_json()?,
6818                        result_envelope,
6819                    )
6820                    .await
6821                {
6822                    if !query_task_rejection_is_final(&error) {
6823                        return Err(error);
6824                    }
6825                }
6826            }
6827            Err(failure) => {
6828                let result = self
6829                    .client
6830                    .fail_query_task(
6831                        &query_task_id,
6832                        &lease_owner,
6833                        attempt,
6834                        failure.message,
6835                        failure.reason,
6836                        failure.failure_type,
6837                    )
6838                    .await;
6839                if let Err(error) = result {
6840                    if !query_task_rejection_is_final(&error) {
6841                        return Err(error);
6842                    }
6843                }
6844            }
6845        }
6846
6847        Ok(ManagedPollOutcome::Handled)
6848    }
6849
6850    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6851        while !stop.load(Ordering::SeqCst) {
6852            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
6853                stop.store(true, Ordering::SeqCst);
6854                break;
6855            }
6856        }
6857
6858        Ok(())
6859    }
6860
6861    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
6862    where
6863        F: FnMut() -> Fut,
6864        Fut: Future<Output = Result<T>>,
6865    {
6866        let mut retries = 0;
6867
6868        loop {
6869            match operation().await {
6870                Err(error)
6871                    if worker_operation_is_retryable(&error)
6872                        && retries < self.retry_policy.max_retries =>
6873                {
6874                    retries += 1;
6875                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
6876                }
6877                result => return result,
6878            }
6879        }
6880    }
6881
6882    async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
6883        match response {
6884            Ok(response) => Ok(Some(response)),
6885            Err(error) => {
6886                let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
6887                    return Err(error);
6888                };
6889                let minimum_delay = self
6890                    .retry_policy
6891                    .initial_backoff
6892                    .max(Duration::from_millis(1));
6893                let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
6894                tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
6895                Ok(None)
6896            }
6897        }
6898    }
6899
6900    async fn execute_query_task(
6901        &self,
6902        mut task: QueryTask,
6903    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
6904        validate_query_task_payloads(&task).map_err(|error| {
6905            QueryTaskExecutionFailure::new(
6906                "query_payload_decode_failed",
6907                error.to_string(),
6908                "QueryPayloadDecodeFailed",
6909            )
6910        })?;
6911
6912        if !self.workflows.contains_key(&task.workflow_type) {
6913            return Err(QueryTaskExecutionFailure::new(
6914                "query_workflow_type_not_registered",
6915                format!("no workflow registered for type {:?}", task.workflow_type),
6916                "WorkflowTypeNotRegistered",
6917            ));
6918        }
6919
6920        let Some(handlers) = self.queries.get(&task.workflow_type) else {
6921            return Err(QueryTaskExecutionFailure::new(
6922                "query_handler_unavailable",
6923                format!(
6924                    "query handlers are unavailable for workflow type {:?}",
6925                    task.workflow_type
6926                ),
6927                "QueryHandlerUnavailable",
6928            ));
6929        };
6930        let Some(query) = handlers.get(&task.query_name) else {
6931            return Err(QueryTaskExecutionFailure::new(
6932                "rejected_unknown_query",
6933                format!("unknown query {:?}", task.query_name),
6934                "QueryFailed",
6935            ));
6936        };
6937
6938        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
6939            .map_err(|error| {
6940            QueryTaskExecutionFailure::new(
6941                "query_payload_decode_failed",
6942                format!("cannot decode query arguments: {error}"),
6943                "QueryPayloadDecodeFailed",
6944            )
6945        })?;
6946        let workflow_input_typed =
6947            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
6948                .map_err(|error| {
6949                    QueryTaskExecutionFailure::new(
6950                        "query_workflow_state_unavailable",
6951                        format!("cannot decode workflow start input: {error}"),
6952                        "QueryWorkflowStateUnavailable",
6953                    )
6954                })?;
6955        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
6956            QueryTaskExecutionFailure::new(
6957                "query_workflow_state_unavailable",
6958                format!("cannot project workflow start input: {error}"),
6959                "QueryWorkflowStateUnavailable",
6960            )
6961        })?;
6962        hydrate_query_history_from_export(&mut task).map_err(|error| {
6963            QueryTaskExecutionFailure::new(
6964                "query_workflow_state_unavailable",
6965                format!("cannot restore query history snapshot: {error}"),
6966                "QueryWorkflowStateUnavailable",
6967            )
6968        })?;
6969        enrich_query_history_from_export(&mut task).map_err(|error| {
6970            QueryTaskExecutionFailure::new(
6971                "query_workflow_state_unavailable",
6972                format!("cannot restore compact query history payloads: {error}"),
6973                "QueryWorkflowStateUnavailable",
6974            )
6975        })?;
6976        let signal_events = query_signal_events(&task).map_err(|error| {
6977            QueryTaskExecutionFailure::new(
6978                "query_workflow_state_unavailable",
6979                format!("cannot decode committed workflow signals: {error}"),
6980                "QueryWorkflowStateUnavailable",
6981            )
6982        })?;
6983        let history_events = Arc::new(std::mem::take(&mut task.history_events));
6984        let context = QueryContext {
6985            workflow_id: task.workflow_id,
6986            run_id: task.run_id,
6987            workflow_type: task.workflow_type.clone(),
6988            run_status: task.run_status,
6989            workflow_input,
6990            workflow_input_avro_value: workflow_input_typed.clone(),
6991            history_events: Arc::clone(&history_events),
6992            signal_events: Arc::new(signal_events),
6993        };
6994
6995        let future = match query {
6996            RegisteredQuery::Snapshot(handler) => handler(context, args),
6997            RegisteredQuery::Replayed {
6998                state_type,
6999                handler,
7000            } => {
7001                let workflow = self
7002                    .workflows
7003                    .get(&task.workflow_type)
7004                    .expect("workflow registration was checked above");
7005                if workflow.state_type != Some(*state_type) {
7006                    return Err(QueryTaskExecutionFailure::new(
7007                        "query_workflow_state_unavailable",
7008                        "replayed query state type does not match its workflow registration",
7009                        "QueryWorkflowStateUnavailable",
7010                    ));
7011                }
7012                let replay = workflow.replay.as_ref().ok_or_else(|| {
7013                    QueryTaskExecutionFailure::new(
7014                        "query_workflow_state_unavailable",
7015                        format!(
7016                            "workflow type {:?} is not registered for instance-state replay",
7017                            task.workflow_type
7018                        ),
7019                        "QueryWorkflowStateUnavailable",
7020                    )
7021                })?;
7022                let workflow_state = Arc::new(Mutex::new(
7023                    WorkflowState::new_with_identity(
7024                        history_events.as_ref().clone(),
7025                        context.workflow_id.clone(),
7026                        context.run_id.clone(),
7027                        self.task_queue.clone(),
7028                        task.payload_codec,
7029                        None,
7030                    )
7031                    .map_err(|error| {
7032                        QueryTaskExecutionFailure::new(
7033                            "query_workflow_state_unavailable",
7034                            format!("workflow replay failed before query: {error}"),
7035                            "QueryWorkflowStateUnavailable",
7036                        )
7037                    })?,
7038                ));
7039                let workflow_context = WorkflowContext {
7040                    state: workflow_state,
7041                };
7042                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7043                let mut cx = TaskContext::from_waker(noop_waker_ref());
7044                match invocation.future.as_mut().poll(&mut cx) {
7045                    Poll::Ready(Ok(_)) => {
7046                        workflow_context
7047                            .ensure_history_consumed()
7048                            .map_err(|error| {
7049                                QueryTaskExecutionFailure::new(
7050                                    "query_workflow_state_unavailable",
7051                                    format!("workflow replay failed before query: {error}"),
7052                                    "QueryWorkflowStateUnavailable",
7053                                )
7054                            })?;
7055                    }
7056                    Poll::Ready(Err(error)) => {
7057                        return Err(QueryTaskExecutionFailure::new(
7058                            "query_workflow_state_unavailable",
7059                            format!("workflow replay failed before query: {error}"),
7060                            "QueryWorkflowStateUnavailable",
7061                        ));
7062                    }
7063                    Poll::Pending => {
7064                        let commands = workflow_context.take_commands().map_err(|error| {
7065                            QueryTaskExecutionFailure::new(
7066                                "query_workflow_state_unavailable",
7067                                format!("workflow replay failed before query: {error}"),
7068                                "QueryWorkflowStateUnavailable",
7069                            )
7070                        })?;
7071                        if commands.is_empty()
7072                            && !workflow_context
7073                                .matched_recorded_pending()
7074                                .map_err(|error| {
7075                                    QueryTaskExecutionFailure::new(
7076                                        "query_workflow_state_unavailable",
7077                                        format!("workflow replay failed before query: {error}"),
7078                                        "QueryWorkflowStateUnavailable",
7079                                    )
7080                                })?
7081                        {
7082                            return Err(QueryTaskExecutionFailure::new(
7083                                "query_workflow_state_unavailable",
7084                                "workflow replay yielded without a durable command",
7085                                "QueryWorkflowStateUnavailable",
7086                            ));
7087                        }
7088                    }
7089                }
7090                let state = (invocation.snapshot)().map_err(|error| {
7091                    QueryTaskExecutionFailure::new(
7092                        "query_workflow_state_unavailable",
7093                        format!("cannot snapshot replayed workflow state: {error}"),
7094                        "QueryWorkflowStateUnavailable",
7095                    )
7096                })?;
7097                handler(context, state, args).map_err(|message| {
7098                    QueryTaskExecutionFailure::new(
7099                        "query_workflow_state_unavailable",
7100                        message,
7101                        "QueryWorkflowStateUnavailable",
7102                    )
7103                })?
7104            }
7105        };
7106
7107        future.await.map_err(|error| {
7108            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7109        })
7110    }
7111
7112    #[cfg(test)]
7113    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7114        Ok(self.execute_workflow_task_decision(task)?.commands)
7115    }
7116
7117    fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7118        validate_workflow_task_payloads(&task)?;
7119
7120        if let Some(update_id) = task
7121            .workflow_update_id
7122            .as_deref()
7123            .filter(|update_id| !update_id.is_empty())
7124        {
7125            return self
7126                .execute_update_task(&task, update_id)
7127                .map(WorkflowTaskDecision::without_message_streams);
7128        }
7129
7130        let workflow = self
7131            .workflows
7132            .get(&task.workflow_type)
7133            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7134        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7135        let resume_signal = decode_resume_signal(&task)?;
7136        let history_budget = WorkflowHistoryBudget {
7137            event_count: task
7138                .total_history_events
7139                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7140            size_bytes: task.history_size_bytes,
7141            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7142            pressure: task.history_budget_pressure.clone(),
7143        };
7144        let workflow_command_identity = task
7145            .workflow_command_id
7146            .clone()
7147            .filter(|identity| !identity.is_empty())
7148            .unwrap_or_default();
7149        let mut workflow_state = WorkflowState::new_with_identity(
7150            task.history_events,
7151            task.workflow_id,
7152            task.run_id,
7153            self.task_queue.clone(),
7154            task.payload_codec.clone(),
7155            resume_signal,
7156        )?;
7157        workflow_state.history_budget = history_budget;
7158        workflow_state.workflow_command_identity = workflow_command_identity;
7159        workflow_state.cancel_requested = task.cancel_requested;
7160        let state = Arc::new(Mutex::new(workflow_state));
7161        let ctx = WorkflowContext { state };
7162        let mut future = (workflow.execute)(ctx.clone(), input);
7163        let mut cx = TaskContext::from_waker(noop_waker_ref());
7164
7165        match future.as_mut().poll(&mut cx) {
7166            Poll::Ready(Ok(result)) => {
7167                ctx.ensure_history_consumed()?;
7168                let result = encode_typed_envelope(&result, &task.payload_codec)?;
7169                let mut commands = ctx.take_commands()?;
7170                commands.push(json!({
7171                    "type": "complete_workflow",
7172                    "result": result
7173                }));
7174                self.message_stream_decision(&ctx, commands)
7175            }
7176            Poll::Ready(Err(error)) => {
7177                if let Error::ContinueAsNew(request) = error {
7178                    let mut commands = ctx.take_commands()?;
7179                    if let Some(command) = ctx.continue_as_new_command(request)? {
7180                        commands.push(command);
7181                    }
7182                    ctx.ensure_history_consumed()?;
7183                    return self.message_stream_decision(&ctx, commands);
7184                }
7185                if workflow_task_integrity_error(&error) {
7186                    // Replay and protocol failures describe the workflow-task
7187                    // decision itself. Preserve their specific failure reason
7188                    // instead of replacing it with the derivative fact that
7189                    // recorded commands remain unconsumed.
7190                    return Err(error);
7191                }
7192                // A handler error must not hide a committed durable command that
7193                // upgraded workflow code no longer consumes.
7194                ctx.ensure_history_consumed()?;
7195                let mut commands = ctx.take_commands()?;
7196                commands.push(workflow_failure_command(&error));
7197                self.message_stream_decision(&ctx, commands)
7198            }
7199            Poll::Pending => {
7200                let commands = ctx.take_commands()?;
7201                if commands.is_empty() && !ctx.matched_recorded_pending()? {
7202                    Err(Error::WorkflowYieldedWithoutCommand)
7203                } else {
7204                    self.message_stream_decision(&ctx, commands)
7205                }
7206            }
7207        }
7208    }
7209
7210    fn message_stream_decision(
7211        &self,
7212        ctx: &WorkflowContext,
7213        commands: Vec<Value>,
7214    ) -> Result<WorkflowTaskDecision> {
7215        let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7216        Ok(WorkflowTaskDecision {
7217            commands,
7218            message_stream_cursors,
7219            message_stream_waits,
7220        })
7221    }
7222
7223    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7224        if !self.workflows.contains_key(&task.workflow_type) {
7225            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7226        }
7227
7228        let accepted = task.history_events.iter().rev().find_map(|event| {
7229            (event.event_type == "UpdateAccepted"
7230                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7231            .then_some(&event.payload)
7232        });
7233        let update_name = accepted
7234            .and_then(|payload| payload.get("update_name"))
7235            .and_then(Value::as_str)
7236            .or(task.update_name.as_deref())
7237            .unwrap_or_default();
7238        let Some(handler) = self
7239            .updates
7240            .get(&task.workflow_type)
7241            .and_then(|handlers| handlers.get(update_name))
7242        else {
7243            return Ok(vec![json!({
7244                "type": "fail_update",
7245                "update_id": update_id,
7246                "message": format!(
7247                    "no update handler is registered for {}.{update_name}",
7248                    task.workflow_type
7249                ),
7250                "exception_type": "UnknownUpdate",
7251                "non_retryable": true,
7252            })]);
7253        };
7254        let arguments = accepted
7255            .and_then(|payload| payload.get("arguments"))
7256            .or(task.arguments.as_ref());
7257        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7258        let context = QueryContext {
7259            workflow_id: task.workflow_id.clone(),
7260            run_id: task.run_id.clone(),
7261            workflow_type: task.workflow_type.clone(),
7262            run_status: Some("running".to_string()),
7263            workflow_input: Value::Null,
7264            workflow_input_avro_value: AvroValue::Null,
7265            history_events: Arc::new(task.history_events.clone()),
7266            signal_events: Arc::new(Vec::new()),
7267        };
7268        let mut future = handler(context, arguments);
7269        let mut cx = TaskContext::from_waker(noop_waker_ref());
7270
7271        match future.as_mut().poll(&mut cx) {
7272            Poll::Ready(Ok(result)) => Ok(vec![json!({
7273                "type": "complete_update",
7274                "update_id": update_id,
7275                "result": encode_typed_envelope(&result, &task.payload_codec)?,
7276            })]),
7277            Poll::Ready(Err(error)) => Ok(vec![json!({
7278                "type": "fail_update",
7279                "update_id": update_id,
7280                "message": error.to_string(),
7281                "exception_type": "UpdateFailed",
7282                "non_retryable": true,
7283            })]),
7284            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7285        }
7286    }
7287
7288    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7289        validate_activity_task_payloads(&task)?;
7290
7291        let handler = self
7292            .activities
7293            .get(&task.activity_type)
7294            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7295        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7296        let attempt_id = task
7297            .activity_attempt_id
7298            .clone()
7299            .or(task.attempt_id.clone())
7300            .unwrap_or_default();
7301        let lease_owner = task
7302            .lease_owner
7303            .clone()
7304            .unwrap_or_else(|| self.worker_id.clone());
7305        let ctx = ActivityContext {
7306            client: self.client.clone(),
7307            task_id: task.task_id,
7308            activity_attempt_id: attempt_id,
7309            lease_owner,
7310            activity_type: task.activity_type,
7311            attempt_number: task.attempt_number,
7312            task_queue: self.task_queue.clone(),
7313            worker_id: self.worker_id.clone(),
7314        };
7315
7316        handler(ctx, args).await
7317    }
7318}
7319
7320fn poller_result(
7321    kind: &str,
7322    result: std::result::Result<Result<()>, tokio::task::JoinError>,
7323) -> Result<()> {
7324    match result {
7325        Ok(result) => result,
7326        Err(error) => Err(Error::WorkerLoop(format!(
7327            "{kind} poller join error: {error}"
7328        ))),
7329    }
7330}
7331
7332fn optional_poller_result(
7333    kind: &str,
7334    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7335) -> Result<()> {
7336    match result {
7337        Some(result) => poller_result(kind, result),
7338        None => Ok(()),
7339    }
7340}
7341
7342async fn join_pollers(
7343    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7344    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7345    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7346) -> Result<()> {
7347    let mut first_error = None;
7348
7349    if let Some(handle) = workflow_poller {
7350        if let Err(error) = poller_result("workflow", handle.await) {
7351            first_error.get_or_insert(error);
7352        }
7353    }
7354
7355    if let Some(handle) = activity_poller {
7356        if let Err(error) = poller_result("activity", handle.await) {
7357            first_error.get_or_insert(error);
7358        }
7359    }
7360
7361    if let Some(handle) = query_poller {
7362        if let Err(error) = poller_result("query", handle.await) {
7363            first_error.get_or_insert(error);
7364        }
7365    }
7366
7367    if let Some(error) = first_error {
7368        Err(error)
7369    } else {
7370        Ok(())
7371    }
7372}
7373
7374fn default_worker_id() -> String {
7375    let millis = SystemTime::now()
7376        .duration_since(UNIX_EPOCH)
7377        .unwrap_or_default()
7378        .as_millis();
7379    format!("rust-worker-{}-{millis}", std::process::id())
7380}
7381
7382fn percent_encode_path_segment(segment: &str) -> String {
7383    const HEX: &[u8; 16] = b"0123456789ABCDEF";
7384    let mut encoded = String::with_capacity(segment.len());
7385
7386    for byte in segment.bytes() {
7387        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7388            encoded.push(char::from(byte));
7389        } else {
7390            encoded.push('%');
7391            encoded.push(char::from(HEX[(byte >> 4) as usize]));
7392            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7393        }
7394    }
7395
7396    encoded
7397}
7398
7399fn unique_request_id(prefix: &str) -> String {
7400    let nanos = SystemTime::now()
7401        .duration_since(UNIX_EPOCH)
7402        .unwrap_or_default()
7403        .as_nanos();
7404    format!("{prefix}-{}-{nanos}", std::process::id())
7405}
7406
7407#[derive(Debug)]
7408struct QueryTaskExecutionFailure {
7409    reason: String,
7410    message: String,
7411    failure_type: String,
7412}
7413
7414impl QueryTaskExecutionFailure {
7415    fn new(
7416        reason: impl Into<String>,
7417        message: impl Into<String>,
7418        failure_type: impl Into<String>,
7419    ) -> Self {
7420        Self {
7421            reason: reason.into(),
7422            message: message.into(),
7423            failure_type: failure_type.into(),
7424        }
7425    }
7426}
7427
7428/// Typed local state owned by one deterministic workflow invocation.
7429///
7430/// Use [`WorkflowInstance::update`] for the same state transitions during
7431/// ordinary execution and replay. A replayed query receives a detached
7432/// immutable `Arc<S>` rather than this mutation-capable handle.
7433#[derive(Clone, Debug)]
7434pub struct WorkflowInstance<S> {
7435    state: Arc<Mutex<S>>,
7436}
7437
7438impl<S> WorkflowInstance<S> {
7439    fn new(state: S) -> Self {
7440        Self {
7441            state: Arc::new(Mutex::new(state)),
7442        }
7443    }
7444
7445    /// Read the current workflow-instance state without changing it.
7446    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7447        let state = self
7448            .state
7449            .lock()
7450            .map_err(|_| Error::WorkflowStatePoisoned)?;
7451        Ok(reader(&state))
7452    }
7453
7454    /// Apply one deterministic workflow-instance state transition.
7455    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7456        let mut state = self
7457            .state
7458            .lock()
7459            .map_err(|_| Error::WorkflowStatePoisoned)?;
7460        Ok(transition(&mut state))
7461    }
7462}
7463
7464impl<S: Clone> WorkflowInstance<S> {
7465    fn snapshot(&self) -> Result<S> {
7466        self.read(Clone::clone)
7467    }
7468}
7469
7470#[derive(Clone, Debug, PartialEq)]
7471pub struct MessageStreamMessage {
7472    pub stream_name: String,
7473    pub message_id: String,
7474    pub position: u64,
7475    pub arguments: Vec<AvroValue>,
7476}
7477
7478#[derive(Clone, Debug)]
7479pub struct MessageStream {
7480    ctx: WorkflowContext,
7481    name: String,
7482}
7483
7484impl MessageStream {
7485    /// Wait for one message, then return a bounded currently-available batch.
7486    pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7487        if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7488            return Err(Error::Codec(format!(
7489                "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7490            )));
7491        }
7492        loop {
7493            if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7494                return Ok(batch);
7495            }
7496
7497            self.ctx.record_message_stream_wait(&self.name)?;
7498            let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7499            let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7500            self.ctx.buffer_message_stream_delivery(arguments)?;
7501            if let Some(sequence) = replay_wait_sequence {
7502                self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7503            }
7504        }
7505    }
7506
7507    pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7508        self.receive(1)
7509            .await?
7510            .into_iter()
7511            .next()
7512            .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7513    }
7514}
7515
7516#[derive(Clone, Debug)]
7517pub struct WorkflowContext {
7518    state: Arc<Mutex<WorkflowState>>,
7519}
7520
7521fn valid_memo_key(key: &str) -> bool {
7522    let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7523
7524    !key.is_empty()
7525        && key.len() <= 64
7526        && (numeric_candidate.is_empty()
7527            || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7528        && key
7529            .bytes()
7530            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7531}
7532
7533fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7534    BASE64
7535        .decode(encode_avro_value(value)?.blob)
7536        .map(|bytes| bytes.len())
7537        .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7538}
7539
7540fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7541    let AvroValue::Map(entries) = value else {
7542        return Err(Error::InvalidMemoUpdate(
7543            "entries must serialize to an Avro string-keyed map".to_string(),
7544        ));
7545    };
7546    if require_entries && entries.is_empty() {
7547        return Err(Error::InvalidMemoUpdate(
7548            "at least one entry is required".to_string(),
7549        ));
7550    }
7551    if entries.len() > MAX_MEMO_ENTRIES {
7552        return Err(Error::InvalidMemoUpdate(format!(
7553            "at most {MAX_MEMO_ENTRIES} entries are allowed"
7554        )));
7555    }
7556
7557    for (key, value) in &entries {
7558        if !valid_memo_key(&key) {
7559            return Err(Error::InvalidMemoUpdate(
7560                "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7561            ));
7562        }
7563        if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7564            return Err(Error::InvalidMemoUpdate(format!(
7565                "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7566            )));
7567        }
7568    }
7569
7570    let value = AvroValue::Map(entries);
7571    if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7572        return Err(Error::InvalidMemoUpdate(format!(
7573            "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7574        )));
7575    }
7576    Ok(value)
7577}
7578
7579fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7580    let object = envelope.as_object().ok_or_else(|| {
7581        Error::InvalidMemoUpdate(
7582            "history field must use the public {codec, blob} payload envelope".to_string(),
7583        )
7584    })?;
7585    if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7586        return Err(Error::InvalidMemoUpdate(
7587            "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7588        ));
7589    }
7590
7591    canonical_memo_entries(
7592        decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7593        require_entries,
7594    )
7595}
7596
7597impl WorkflowContext {
7598    pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7599        let name = name.into();
7600        if name.is_empty()
7601            || name.len() > 128
7602            || !name.bytes().all(|byte| {
7603                byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7604            })
7605        {
7606            return Err(Error::Codec(
7607                "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7608                    .to_string(),
7609            ));
7610        }
7611        Ok(MessageStream {
7612            ctx: self.clone(),
7613            name,
7614        })
7615    }
7616
7617    fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7618        let mut state = self
7619            .state
7620            .lock()
7621            .map_err(|_| Error::WorkflowStatePoisoned)?;
7622        let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7623        state
7624            .message_stream_waits
7625            .insert(name.to_string(), position);
7626        Ok(())
7627    }
7628
7629    fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7630        let mut state = self
7631            .state
7632            .lock()
7633            .map_err(|_| Error::WorkflowStatePoisoned)?;
7634        let cursor = state
7635            .message_stream_cursors
7636            .get(&message.stream_name)
7637            .copied()
7638            .unwrap_or(0);
7639        if message.position <= cursor {
7640            return Ok(());
7641        }
7642        let pending = state
7643            .message_stream_messages
7644            .entry(message.stream_name.clone())
7645            .or_default();
7646        if pending.iter().any(|candidate| {
7647            candidate.position == message.position || candidate.message_id == message.message_id
7648        }) {
7649            return Ok(());
7650        }
7651        pending.push(message);
7652        pending.sort_by_key(|candidate| candidate.position);
7653        Ok(())
7654    }
7655
7656    fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7657        if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7658            match delivery {
7659                MessageStreamDelivery::Message(message) => {
7660                    let stream_name = message.stream_name.clone();
7661                    self.buffer_message_stream(message)?;
7662                    return Ok(Some(stream_name));
7663                }
7664                MessageStreamDelivery::Cursor {
7665                    stream_name,
7666                    through_position,
7667                } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7668            }
7669        }
7670        Ok(None)
7671    }
7672
7673    fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7674        let state = self
7675            .state
7676            .lock()
7677            .map_err(|_| Error::WorkflowStatePoisoned)?;
7678        Ok(match state.recorded_commands.get(state.command_cursor) {
7679            Some(RecordedCommand::SignalWait {
7680                sequence,
7681                signal_name,
7682                ..
7683            }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7684            _ => None,
7685        })
7686    }
7687
7688    fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7689        let (history, payload_codec) = {
7690            let state = self
7691                .state
7692                .lock()
7693                .map_err(|_| Error::WorkflowStatePoisoned)?;
7694            (
7695                Arc::clone(&state.history_events),
7696                state.payload_codec.clone(),
7697            )
7698        };
7699
7700        let Some(opened_index) = history.iter().position(|event| {
7701            event.event_type == "SignalWaitOpened"
7702                && durable_event_sequence(event) == Some(wait_sequence)
7703                && event.payload.get("signal_name").and_then(Value::as_str)
7704                    == Some(MESSAGE_STREAM_SIGNAL)
7705        }) else {
7706            return Ok(());
7707        };
7708        let boundary_index = history
7709            .iter()
7710            .enumerate()
7711            .skip(opened_index + 1)
7712            .find_map(|(index, event)| {
7713                (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7714                    && is_authored_command_open_event(event))
7715                .then_some(index)
7716            })
7717            .unwrap_or(history.len());
7718
7719        for event in history[opened_index + 1..boundary_index]
7720            .iter()
7721            .filter(|event| {
7722                event.event_type == "SignalReceived"
7723                    && event.payload.get("signal_name").and_then(Value::as_str)
7724                        == Some(MESSAGE_STREAM_SIGNAL)
7725            })
7726        {
7727            let arguments = decode_signal_event_arguments(event, &payload_codec)?
7728                .into_iter()
7729                .map(AvroValue::into_json)
7730                .collect::<Result<Vec<_>>>()?;
7731            self.buffer_message_stream_delivery(arguments)?;
7732        }
7733        Ok(())
7734    }
7735
7736    fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7737        let mut state = self
7738            .state
7739            .lock()
7740            .map_err(|_| Error::WorkflowStatePoisoned)?;
7741        let cursor = state
7742            .message_stream_cursors
7743            .entry(name.to_string())
7744            .or_default();
7745        *cursor = (*cursor).max(through_position);
7746        if let Some(pending) = state.message_stream_messages.get_mut(name) {
7747            pending.retain(|message| message.position > through_position);
7748        }
7749        Ok(())
7750    }
7751
7752    fn take_message_stream_batch(
7753        &self,
7754        name: &str,
7755        max_items: usize,
7756    ) -> Result<Option<Vec<MessageStreamMessage>>> {
7757        let mut state = self
7758            .state
7759            .lock()
7760            .map_err(|_| Error::WorkflowStatePoisoned)?;
7761        let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7762        let pending = state
7763            .message_stream_messages
7764            .entry(name.to_string())
7765            .or_default();
7766        let count = contiguous_message_stream_count(pending, cursor, max_items);
7767        if count == 0 {
7768            return Ok(None);
7769        }
7770        let batch = pending.drain(..count).collect::<Vec<_>>();
7771        let position = batch.last().map(|message| message.position).unwrap_or(0);
7772        state
7773            .message_stream_cursors
7774            .insert(name.to_string(), position);
7775        state.message_stream_waits.remove(name);
7776        Ok(Some(batch))
7777    }
7778
7779    fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7780        let state = self
7781            .state
7782            .lock()
7783            .map_err(|_| Error::WorkflowStatePoisoned)?;
7784        let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7785        cursors.sort_by_key(|(name, _)| *name);
7786        let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7787        waits.sort_by_key(|(name, _)| *name);
7788        Ok((
7789            cursors
7790                .into_iter()
7791                .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7792                .collect(),
7793            waits
7794                .into_iter()
7795                .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7796                .collect(),
7797        ))
7798    }
7799    /// Identity of the parent workflow currently being replayed.
7800    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7801        let state = self
7802            .state
7803            .lock()
7804            .map_err(|_| Error::WorkflowStatePoisoned)?;
7805        Ok(WorkflowIdentity {
7806            workflow_id: state.workflow_id.clone(),
7807            run_id: state.run_id.clone(),
7808        })
7809    }
7810
7811    /// Return the server-published history budget for this workflow task.
7812    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7813        let state = self
7814            .state
7815            .lock()
7816            .map_err(|_| Error::WorkflowStatePoisoned)?;
7817        Ok(state.history_budget.clone())
7818    }
7819
7820    /// Continue this workflow instance as a fresh run with replacement arguments.
7821    ///
7822    /// Return this value directly from the workflow handler. The worker converts
7823    /// it to the terminal protocol command only after replay has consumed every
7824    /// recorded durable command.
7825    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7826        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7827    }
7828
7829    /// Continue as new with optional workflow-type and task-queue overrides.
7830    pub fn continue_as_new_with_options<T: Serialize>(
7831        &self,
7832        options: ContinueAsNewOptions,
7833        args: T,
7834    ) -> Result<Value> {
7835        options.validate()?;
7836        Err(Error::ContinueAsNew(ContinueAsNewRequest {
7837            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
7838            options,
7839        }))
7840    }
7841
7842    pub fn activity<T: Serialize>(
7843        &self,
7844        activity_type: impl Into<String>,
7845        args: T,
7846    ) -> ActivityCall {
7847        self.activity_with_options(activity_type, ActivityOptions::new(), args)
7848    }
7849
7850    pub fn activity_on_queue<T, Q>(
7851        &self,
7852        activity_type: impl Into<String>,
7853        task_queue: Option<Q>,
7854        args: T,
7855    ) -> ActivityCall
7856    where
7857        T: Serialize,
7858        Q: Into<String>,
7859    {
7860        let mut options = ActivityOptions::new();
7861        options.task_queue = task_queue.map(Into::into);
7862        self.activity_with_options(activity_type, options, args)
7863    }
7864
7865    /// Schedule one durable activity with retry, routing, and timeout options.
7866    ///
7867    /// Options are validated before the command is emitted. Once the command is
7868    /// recorded, replay consumes the same activity lifecycle at this command
7869    /// position and never emits a duplicate schedule.
7870    ///
7871    /// ```no_run
7872    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
7873    /// # use std::time::Duration;
7874    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
7875    /// let result = ctx
7876    ///     .activity_with_options(
7877    ///         "charge-card",
7878    ///         ActivityOptions::new()
7879    ///             .task_queue("payments")
7880    ///             .retry_policy(
7881    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
7882    ///                     Duration::from_secs(1),
7883    ///                     2,
7884    ///                     Some(Duration::from_secs(30)),
7885    ///                 ),
7886    ///             )
7887    ///             .start_to_close_timeout(Duration::from_secs(60))
7888    ///             .schedule_to_close_timeout(Duration::from_secs(180))
7889    ///             .heartbeat_timeout(Duration::from_secs(15)),
7890    ///         json!([{"order_id": "order-42"}]),
7891    ///     )
7892    ///     .await;
7893    /// match result {
7894    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
7895    ///         "reason": failure.reason,
7896    ///         "timeout_kind": failure.timeout_kind,
7897    ///     })),
7898    ///     other => other,
7899    /// }
7900    /// # }
7901    /// ```
7902    pub fn activity_with_options<T: Serialize>(
7903        &self,
7904        activity_type: impl Into<String>,
7905        options: ActivityOptions,
7906        args: T,
7907    ) -> ActivityCall {
7908        ActivityCall {
7909            ctx: self.clone(),
7910            activity_type: activity_type.into(),
7911            options,
7912            args: Some(AvroValue::from_serialize(&args)),
7913            scheduled: false,
7914            parallel_group_path: Vec::new(),
7915        }
7916    }
7917
7918    pub async fn activity_avro_value<T: Serialize>(
7919        &self,
7920        activity_type: impl Into<String>,
7921        args: T,
7922    ) -> Result<AvroValue> {
7923        let mut call = self.activity(activity_type, args);
7924        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7925    }
7926
7927    pub async fn activity_avro_value_with_options<T: Serialize>(
7928        &self,
7929        activity_type: impl Into<String>,
7930        options: ActivityOptions,
7931        args: T,
7932    ) -> Result<AvroValue> {
7933        let mut call = self.activity_with_options(activity_type, options, args);
7934        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7935    }
7936
7937    /// Schedule an activity with a Serde request and decode its Serde result.
7938    pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
7939    where
7940        I: Serialize,
7941        O: DeserializeOwned,
7942    {
7943        self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
7944            .await
7945    }
7946
7947    /// Schedule an activity with options and decode its result into `O`.
7948    ///
7949    /// Both directions use the fixed Avro Value codec. In particular, this
7950    /// method does not deserialize the JSON-safe inspection projection returned
7951    /// by the dynamic [`ActivityCall`] future.
7952    pub async fn activity_typed_with_options<I, O>(
7953        &self,
7954        activity_type: impl Into<String>,
7955        options: ActivityOptions,
7956        args: I,
7957    ) -> Result<O>
7958    where
7959        I: Serialize,
7960        O: DeserializeOwned,
7961    {
7962        let activity_type = activity_type.into();
7963        let encoded = AvroValue::from_serialize(&args).map_err(|error| {
7964            handler_type_error::<I>(
7965                HandlerKind::Activity,
7966                &activity_type,
7967                HandlerValueKind::Input,
7968                error.to_string(),
7969            )
7970        });
7971        let mut call = ActivityCall {
7972            ctx: self.clone(),
7973            activity_type: activity_type.clone(),
7974            options,
7975            args: Some(encoded),
7976            scheduled: false,
7977            parallel_group_path: Vec::new(),
7978        };
7979        let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
7980        decode_handler_result(result, HandlerKind::Activity, &activity_type)
7981    }
7982
7983    /// Schedule and join a deterministic activity/child/timer group.
7984    ///
7985    /// Nested groups retain their input shape. Every durable leaf is scheduled
7986    /// before this future yields, results are assembled by declaration order,
7987    /// and a failure returns [`Error::ParallelFailed`] with typed cause,
7988    /// declaration path, stable group metadata, and completed siblings.
7989    pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7990        ParallelCall::new(self.clone(), operations)
7991    }
7992
7993    /// Alias for [`WorkflowContext::parallel`].
7994    pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7995        self.parallel(operations)
7996    }
7997
7998    /// Lossless fixed-Avro variant of [`WorkflowContext::parallel`].
7999    pub async fn parallel_avro_value(
8000        &self,
8001        operations: Vec<ParallelOperation>,
8002    ) -> Result<Vec<ParallelAvroResult>> {
8003        let mut call = self.parallel(operations);
8004        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8005    }
8006
8007    /// Start every durable operation and resume from the one winner persisted
8008    /// by Server. Non-winning operations continue and remain addressable.
8009    pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8010        let operations = operations
8011            .into_iter()
8012            .enumerate()
8013            .map(|(index, operation)| (SelectionKey::Index(index), operation))
8014            .collect();
8015        SelectCall::new(self.clone(), operations)
8016    }
8017
8018    /// Named-key variant of [`WorkflowContext::select`].
8019    pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8020    where
8021        K: Into<SelectionKey>,
8022    {
8023        SelectCall::new(
8024            self.clone(),
8025            operations
8026                .into_iter()
8027                .map(|(key, operation)| (key.into(), operation))
8028                .collect(),
8029        )
8030    }
8031
8032    /// Create a workflow-local deterministic compensation registry.
8033    pub fn saga(&self) -> Saga {
8034        Saga::new(self.clone())
8035    }
8036
8037    /// Whether the current workflow task carries a cooperative cancel request.
8038    pub fn is_cancellation_requested(&self) -> Result<bool> {
8039        let state = self
8040            .state
8041            .lock()
8042            .map_err(|_| Error::WorkflowStatePoisoned)?;
8043        Ok(state.cancel_requested)
8044    }
8045
8046    /// Raise a typed cooperative cancellation at an author-controlled point.
8047    ///
8048    /// Passing this result to [`Saga::finish`] compensates already registered
8049    /// forward steps before the cancellation remains the initiating outcome.
8050    pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8051        if self.is_cancellation_requested()? {
8052            return Err(Error::WorkflowCancellationRequested(
8053                WorkflowCancellationRequested,
8054            ));
8055        }
8056        Ok(())
8057    }
8058
8059    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8060        SignalCall {
8061            ctx: self.clone(),
8062            signal_name: signal_name.into(),
8063            runtime_reserved_allowed: false,
8064            opened_wait: false,
8065            matched_pending: false,
8066            parallel_group_path: Vec::new(),
8067        }
8068    }
8069
8070    fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8071        SignalCall {
8072            ctx: self.clone(),
8073            signal_name: signal_name.into(),
8074            runtime_reserved_allowed: true,
8075            opened_wait: false,
8076            matched_pending: false,
8077            parallel_group_path: Vec::new(),
8078        }
8079    }
8080
8081    pub async fn wait_signal_avro_value(
8082        &self,
8083        signal_name: impl Into<String>,
8084    ) -> Result<Vec<AvroValue>> {
8085        let mut call = self.wait_signal(signal_name);
8086        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8087    }
8088
8089    /// Return every committed signal argument list with the given name.
8090    ///
8091    /// This history-backed view is deterministic and is intended for
8092    /// condition predicates that must be re-evaluated after a signal while the
8093    /// workflow is blocked on [`WorkflowContext::wait_condition`].
8094    pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8095        self.signals_avro_value(signal_name)?
8096            .into_iter()
8097            .map(|arguments| {
8098                arguments
8099                    .into_iter()
8100                    .map(AvroValue::into_json)
8101                    .collect::<Result<Vec<_>>>()
8102            })
8103            .collect()
8104    }
8105
8106    /// Lossless fixed Avro Value view of committed signals with the given name.
8107    pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8108        let state = self
8109            .state
8110            .lock()
8111            .map_err(|_| Error::WorkflowStatePoisoned)?;
8112        state
8113            .history_events
8114            .iter()
8115            .filter(|event| {
8116                event.event_type == "SignalReceived"
8117                    && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8118            })
8119            .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8120            .collect()
8121    }
8122
8123    /// Return every committed update argument list with the given name.
8124    ///
8125    /// Accepted and applied records for the same update ID are de-duplicated.
8126    /// A Server task created after an update therefore replays the workflow and
8127    /// re-evaluates an open condition without application polling.
8128    pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8129        self.updates_avro_value(update_name)?
8130            .into_iter()
8131            .map(|arguments| {
8132                arguments
8133                    .into_iter()
8134                    .map(AvroValue::into_json)
8135                    .collect::<Result<Vec<_>>>()
8136            })
8137            .collect()
8138    }
8139
8140    /// Lossless fixed Avro Value view of committed updates with the given name.
8141    pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8142        let state = self
8143            .state
8144            .lock()
8145            .map_err(|_| Error::WorkflowStatePoisoned)?;
8146        let mut seen = Vec::new();
8147        let mut updates = Vec::new();
8148        for event in state.history_events.iter() {
8149            if !matches!(
8150                event.event_type.as_str(),
8151                "UpdateAccepted" | "UpdateApplied"
8152            ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8153                || event.payload.get("arguments").is_none()
8154            {
8155                continue;
8156            }
8157            if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8158                if seen.iter().any(|recorded| recorded == update_id) {
8159                    continue;
8160                }
8161                seen.push(update_id.to_string());
8162            }
8163            updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8164        }
8165        Ok(updates)
8166    }
8167
8168    /// Wait for a deterministic predicate to become true or for its durable
8169    /// timeout to elapse.
8170    ///
8171    /// Prefer [`wait_condition!`] for inline predicates so changes to the Rust
8172    /// predicate tokens automatically change the recorded definition
8173    /// fingerprint. Direct callers must provide an equally stable identity in
8174    /// [`ConditionWaitOptions`].
8175    pub fn wait_condition<F>(
8176        &self,
8177        options: ConditionWaitOptions,
8178        predicate: F,
8179    ) -> ConditionWaitCall
8180    where
8181        F: Fn() -> Result<bool> + Send + 'static,
8182    {
8183        ConditionWaitCall {
8184            ctx: self.clone(),
8185            options,
8186            predicate: Box::new(predicate),
8187            occurrence_id: None,
8188            opened_wait: false,
8189            parallel_group_path: Vec::new(),
8190        }
8191    }
8192
8193    /// Wait for server-backed durable time without blocking the worker executor.
8194    ///
8195    /// Polling this future emits one `start_timer` command and yields. The
8196    /// server records the deadline, so neither worker nor server restarts reset
8197    /// the wait. Replay resolves the future only from a `TimerScheduled` and
8198    /// `TimerFired` pair at the same position in the shared durable-command
8199    /// stream, with matching sequence, timer identity, and delay. Sub-second
8200    /// durations round up because protocol deadlines use whole seconds.
8201    ///
8202    /// ```no_run
8203    /// # use durable_workflow::{json, Client, Worker};
8204    /// # use std::time::Duration;
8205    /// # fn configure(client: Client) {
8206    /// let mut worker = Worker::new(client, "rust-workers");
8207    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
8208    ///     ctx.sleep(Duration::from_secs(5)).await?;
8209    ///     Ok(json!({"status": "timer fired"}))
8210    /// });
8211    /// # }
8212    /// ```
8213    pub fn sleep(&self, duration: Duration) -> TimerCall {
8214        let delay_seconds = duration
8215            .as_secs()
8216            .checked_add(u64::from(duration.subsec_nanos() > 0));
8217        TimerCall {
8218            ctx: self.clone(),
8219            delay_seconds,
8220            scheduled: false,
8221            matched_pending: false,
8222            parallel_group_path: Vec::new(),
8223        }
8224    }
8225
8226    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
8227    pub fn start_timer(&self, duration: Duration) -> TimerCall {
8228        self.sleep(duration)
8229    }
8230
8231    /// Evaluate a non-deterministic callback once and durably record its typed value.
8232    ///
8233    /// On replay the callback is not invoked: the value is decoded from the
8234    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
8235    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
8236    /// small values that must remain fixed for the lifetime of a workflow run.
8237    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8238    where
8239        T: Serialize + DeserializeOwned,
8240        F: FnOnce() -> T,
8241    {
8242        {
8243            let mut state = self
8244                .state
8245                .lock()
8246                .map_err(|_| Error::WorkflowStatePoisoned)?;
8247            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8248                return match recorded {
8249                    RecordedCommand::SideEffect { sequence, value } => {
8250                        state.command_cursor += 1;
8251                        value.deserialize().map_err(|error| {
8252                            Error::NonDeterministicReplay(ReplayFailure::new(
8253                                "side_effect_type_mismatch",
8254                                Some(sequence),
8255                                Some(std::any::type_name::<T>().to_string()),
8256                                Some(error.to_string()),
8257                                "recorded side-effect value is incompatible with the requested Rust type",
8258                            ))
8259                        })
8260                    }
8261                    other => Err(command_mismatch(&other, "side effect")),
8262                };
8263            }
8264        }
8265
8266        let value = callback();
8267        let avro_value = AvroValue::from_serialize(&value)?;
8268        let mut state = self
8269            .state
8270            .lock()
8271            .map_err(|_| Error::WorkflowStatePoisoned)?;
8272        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8273        state.commands.push(json!({
8274            "type": "record_side_effect",
8275            "result": result,
8276        }));
8277        Ok(value)
8278    }
8279
8280    /// Record or replay a lossless fixed Avro Value side effect.
8281    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8282    where
8283        F: FnOnce() -> AvroValue,
8284    {
8285        {
8286            let mut state = self
8287                .state
8288                .lock()
8289                .map_err(|_| Error::WorkflowStatePoisoned)?;
8290            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8291                return match recorded {
8292                    RecordedCommand::SideEffect { value, .. } => {
8293                        state.command_cursor += 1;
8294                        Ok(value)
8295                    }
8296                    other => Err(command_mismatch(&other, "side effect")),
8297                };
8298            }
8299        }
8300
8301        let value = callback();
8302        let mut state = self
8303            .state
8304            .lock()
8305            .map_err(|_| Error::WorkflowStatePoisoned)?;
8306        let result = encode_typed_envelope(&value, &state.payload_codec)?;
8307        state.commands.push(json!({
8308            "type": "record_side_effect",
8309            "result": result,
8310        }));
8311        Ok(value)
8312    }
8313
8314    /// Append output items at a deterministic workflow command boundary.
8315    ///
8316    /// Stable idempotency keys are derived from the server-provided durable
8317    /// workflow command identity, command ordinal, and item index. Replay
8318    /// consumes the recorded side effect and never emits another append.
8319    pub fn append_workflow_stream(
8320        &self,
8321        stream_name: impl Into<String>,
8322        items: &[WorkflowStreamAppendItem],
8323        max_pending_items: Option<u64>,
8324    ) -> Result<()> {
8325        if items.is_empty() {
8326            return Err(Error::Codec(
8327                "workflow_stream_items_empty: append requires at least one item".to_string(),
8328            ));
8329        }
8330        if max_pending_items == Some(0) {
8331            return Err(Error::Codec(
8332                "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8333                    .to_string(),
8334            ));
8335        }
8336        let stream_name = stream_name.into();
8337        if stream_name.is_empty() {
8338            return Err(Error::Codec(
8339                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8340            ));
8341        }
8342
8343        let mut state = self
8344            .state
8345            .lock()
8346            .map_err(|_| Error::WorkflowStatePoisoned)?;
8347        let command_ordinal = state.workflow_stream_command_counter;
8348        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8349            state.workflow_stream_command_counter += 1;
8350            return match recorded {
8351                RecordedCommand::SideEffect { .. } => {
8352                    state.command_cursor += 1;
8353                    Ok(())
8354                }
8355                other => Err(command_mismatch(&other, "workflow stream append")),
8356            };
8357        }
8358
8359        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8360        state.workflow_stream_command_counter += 1;
8361        let wire_items = items
8362            .iter()
8363            .enumerate()
8364            .map(|(item_index, item)| {
8365                item.wire_value(Some(format!(
8366                    "dw-stream:{identity}:{command_ordinal}:{item_index}"
8367                )))
8368            })
8369            .collect::<Vec<_>>();
8370        let mut directive = json!({
8371            "operation": "append",
8372            "stream_name": stream_name,
8373            "command_identity": identity,
8374            "command_ordinal": command_ordinal,
8375            "items": wire_items,
8376        });
8377        if let Some(max_pending_items) = max_pending_items {
8378            directive["max_pending_items"] = json!(max_pending_items);
8379        }
8380        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8381        state.commands.push(json!({
8382            "type": "record_side_effect",
8383            "result": result,
8384            "workflow_stream": directive,
8385        }));
8386        Ok(())
8387    }
8388
8389    /// Close a run-scoped output stream at a deterministic command boundary.
8390    pub fn close_workflow_stream(
8391        &self,
8392        stream_name: impl Into<String>,
8393        retention_seconds: Option<u64>,
8394    ) -> Result<()> {
8395        self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8396    }
8397
8398    /// Mark a run-scoped output stream errored at a deterministic command boundary.
8399    pub fn error_workflow_stream(
8400        &self,
8401        stream_name: impl Into<String>,
8402        error_reason: impl Into<String>,
8403        retention_seconds: Option<u64>,
8404    ) -> Result<()> {
8405        let error_reason = error_reason.into();
8406        if error_reason.is_empty() {
8407            return Err(Error::Codec(
8408                "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8409            ));
8410        }
8411        self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8412    }
8413
8414    fn finish_workflow_stream(
8415        &self,
8416        stream_name: String,
8417        error_reason: Option<String>,
8418        retention_seconds: Option<u64>,
8419    ) -> Result<()> {
8420        if stream_name.is_empty() {
8421            return Err(Error::Codec(
8422                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8423            ));
8424        }
8425        if retention_seconds == Some(0) {
8426            return Err(Error::Codec(
8427                "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8428            ));
8429        }
8430        let mut state = self
8431            .state
8432            .lock()
8433            .map_err(|_| Error::WorkflowStatePoisoned)?;
8434        let command_ordinal = state.workflow_stream_command_counter;
8435        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8436            state.workflow_stream_command_counter += 1;
8437            return match recorded {
8438                RecordedCommand::SideEffect { .. } => {
8439                    state.command_cursor += 1;
8440                    Ok(())
8441                }
8442                other => Err(command_mismatch(&other, "workflow stream close")),
8443            };
8444        }
8445        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8446        state.workflow_stream_command_counter += 1;
8447        let mut directive = json!({
8448            "operation": if error_reason.is_some() { "error" } else { "close" },
8449            "stream_name": stream_name,
8450            "command_identity": identity,
8451            "command_ordinal": command_ordinal,
8452        });
8453        if let Some(error_reason) = error_reason {
8454            directive["error_reason"] = json!(error_reason);
8455        }
8456        if let Some(retention_seconds) = retention_seconds {
8457            directive["retention_seconds"] = json!(retention_seconds);
8458        }
8459        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8460        state.commands.push(json!({
8461            "type": "record_side_effect",
8462            "result": result,
8463            "workflow_stream": directive,
8464        }));
8465        Ok(())
8466    }
8467
8468    fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8469        let identity = state.workflow_command_identity.as_str();
8470        if identity.is_empty() {
8471            return Err(Error::MissingWorkflowCommandIdentity);
8472        }
8473        Ok(identity)
8474    }
8475
8476    /// Validate, emit, or replay a typed workflow search-attribute update.
8477    ///
8478    /// The command is non-blocking within a workflow decision, but its
8479    /// `SearchAttributesUpserted` event occupies the same deterministic command
8480    /// stream as activities, timers, conditions, and other durable operations.
8481    pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8482        update.validate()?;
8483        let (attributes, attribute_types) = update.into_wire_parts();
8484        let mut state = self
8485            .state
8486            .lock()
8487            .map_err(|_| Error::WorkflowStatePoisoned)?;
8488
8489        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8490            return match recorded {
8491                RecordedCommand::SearchAttributes {
8492                    sequence,
8493                    attributes: recorded_attributes,
8494                    attribute_types: recorded_attribute_types,
8495                } => {
8496                    if recorded_attributes != attributes {
8497                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8498                            "search_attribute_value_mismatch",
8499                            Some(sequence),
8500                            Some(recorded_attributes.to_string()),
8501                            Some(attributes.to_string()),
8502                            "search-attribute values differ from the recorded durable command",
8503                        )));
8504                    }
8505                    if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8506                        if recorded_types != attribute_types {
8507                            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8508                                "search_attribute_type_mismatch",
8509                                Some(sequence),
8510                                Some(json!(recorded_types).to_string()),
8511                                Some(json!(attribute_types).to_string()),
8512                                "search-attribute declared types differ from the recorded durable command",
8513                            )));
8514                        }
8515                    }
8516                    state.command_cursor += 1;
8517                    Ok(())
8518                }
8519                other => Err(command_mismatch(&other, "search-attribute update")),
8520            };
8521        }
8522
8523        let mut command = serde_json::Map::from_iter([
8524            ("type".to_string(), json!("upsert_search_attributes")),
8525            ("attributes".to_string(), attributes),
8526        ]);
8527        if !attribute_types.is_empty() {
8528            command.insert("attribute_types".to_string(), json!(attribute_types));
8529        }
8530        state.commands.push(Value::Object(command));
8531        Ok(())
8532    }
8533
8534    /// Record a UUIDv4 once and return the same UUID on every replay.
8535    pub fn uuid_v4(&self) -> Result<Uuid> {
8536        self.side_effect(Uuid::new_v4)
8537    }
8538
8539    /// Select the newest supported version for a change, or replay the version
8540    /// already committed for that stable change ID.
8541    pub fn get_version(
8542        &self,
8543        change_id: impl Into<String>,
8544        min_supported: i32,
8545        max_supported: i32,
8546    ) -> Result<i32> {
8547        let change_id = change_id.into();
8548        if change_id.trim().is_empty() {
8549            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8550                "version_change_id_invalid",
8551                None,
8552                Some("non-empty change ID".to_string()),
8553                Some(change_id),
8554                "version markers require a stable non-empty change ID",
8555            )));
8556        }
8557        if min_supported > max_supported {
8558            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8559                "version_range_invalid",
8560                None,
8561                Some("min_supported <= max_supported".to_string()),
8562                Some(format!("{min_supported}..={max_supported}")),
8563                "version marker supported range is invalid",
8564            )));
8565        }
8566
8567        let mut state = self
8568            .state
8569            .lock()
8570            .map_err(|_| Error::WorkflowStatePoisoned)?;
8571        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8572            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8573            return Ok(version);
8574        }
8575
8576        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8577            return match recorded {
8578                RecordedCommand::VersionMarker {
8579                    sequence,
8580                    change_id: recorded_change_id,
8581                    version,
8582                    ..
8583                } => {
8584                    if recorded_change_id != change_id {
8585                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8586                            "version_change_id_mismatch",
8587                            Some(sequence),
8588                            Some(recorded_change_id),
8589                            Some(change_id),
8590                            "recorded version marker change ID differs from current workflow code",
8591                        )));
8592                    }
8593                    ensure_version_supported(
8594                        &change_id,
8595                        version,
8596                        min_supported,
8597                        max_supported,
8598                        sequence,
8599                    )?;
8600                    state.command_cursor += 1;
8601                    state.version_markers.insert(change_id, (version, sequence));
8602                    Ok(version)
8603                }
8604                other => Err(command_mismatch(
8605                    &other,
8606                    format!("version marker:{change_id}"),
8607                )),
8608            };
8609        }
8610
8611        let version = max_supported;
8612        state.commands.push(json!({
8613            "type": "record_version_marker",
8614            "change_id": change_id,
8615            "version": version,
8616            "min_supported": min_supported,
8617            "max_supported": max_supported,
8618        }));
8619        // Sequence numbers are assigned by the server. Zero identifies a marker
8620        // selected in this uncommitted decision batch for duplicate-call checks.
8621        state.version_markers.insert(change_id, (version, 0));
8622        Ok(version)
8623    }
8624
8625    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
8626    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8627        Ok(self.get_version(change_id, -1, 1)? == 1)
8628    }
8629
8630    /// Keep a patch marker in history after the legacy branch has been removed.
8631    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8632        self.get_version(change_id, -1, 1).map(|_| ())
8633    }
8634
8635    /// Merge non-indexed workflow memo metadata through durable history.
8636    ///
8637    /// Avro `null` deletes a key. The SDK encodes the complete patch in the
8638    /// public Avro payload envelope consumed by Server and Cloud runtimes.
8639    pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8640        let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8641        let mut state = self
8642            .state
8643            .lock()
8644            .map_err(|_| Error::WorkflowStatePoisoned)?;
8645
8646        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8647            return match recorded {
8648                RecordedCommand::Memo {
8649                    sequence,
8650                    entries: recorded_entries,
8651                } => {
8652                    if recorded_entries != entries {
8653                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8654                            "memo_update_mismatch",
8655                            Some(sequence),
8656                            Some(format!("{recorded_entries:?}")),
8657                            Some(format!("{entries:?}")),
8658                            "recorded memo entries differ from the current workflow update",
8659                        )));
8660                    }
8661                    state.command_cursor += 1;
8662                    Ok(())
8663                }
8664                other => Err(command_mismatch(&other, "memo upsert")),
8665            };
8666        }
8667
8668        let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8669        state.commands.push(json!({
8670            "type": "upsert_memo",
8671            "entries": entries_envelope,
8672        }));
8673        Ok(())
8674    }
8675
8676    /// Start a named durable child on an explicit queue and await its result.
8677    ///
8678    /// The command is recorded in the parent's sequence-ordered durable command
8679    /// stream. Replay keeps a scheduled child pending without emitting another
8680    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
8681    /// values preserve the history payload codec and include both sides of the
8682    /// durable relationship; failures are returned as
8683    /// [`Error::ChildWorkflowFailed`].
8684    ///
8685    /// ```no_run
8686    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
8687    /// # fn configure(client: Client) {
8688    /// let mut worker = Worker::new(client, "parent-workers");
8689    /// worker.register_workflow("order-parent", |ctx, _input| async move {
8690    ///     let child = ctx
8691    ///         .start_child_workflow(
8692    ///             "fulfil-order",
8693    ///             ChildWorkflowOptions::new("fulfilment-workers")
8694    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
8695    ///             json!([{"order_id": "order-42"}]),
8696    ///         )
8697    ///         .await?;
8698    ///     Ok(child.result)
8699    /// });
8700    /// # }
8701    /// ```
8702    pub fn start_child_workflow<T: Serialize>(
8703        &self,
8704        workflow_type: impl Into<String>,
8705        options: ChildWorkflowOptions,
8706        args: T,
8707    ) -> ChildWorkflowCall {
8708        ChildWorkflowCall {
8709            ctx: self.clone(),
8710            workflow_type: workflow_type.into(),
8711            options,
8712            args: Some(AvroValue::from_serialize(&args)),
8713            scheduled: false,
8714            matched_pending: false,
8715            parallel_group_path: Vec::new(),
8716        }
8717    }
8718
8719    pub async fn start_child_workflow_avro_value<T: Serialize>(
8720        &self,
8721        workflow_type: impl Into<String>,
8722        options: ChildWorkflowOptions,
8723        args: T,
8724    ) -> Result<ChildWorkflowAvroResult> {
8725        let mut call = self.start_child_workflow(workflow_type, options, args);
8726        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8727    }
8728
8729    fn take_commands(&self) -> Result<Vec<Value>> {
8730        let mut state = self
8731            .state
8732            .lock()
8733            .map_err(|_| Error::WorkflowStatePoisoned)?;
8734        Ok(std::mem::take(&mut state.commands))
8735    }
8736
8737    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8738        let mut state = self
8739            .state
8740            .lock()
8741            .map_err(|_| Error::WorkflowStatePoisoned)?;
8742
8743        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8744            return Err(command_mismatch(&recorded, "continue as new"));
8745        }
8746        if state.recorded_continue_as_new_sequence.is_some() {
8747            state.continue_as_new_consumed = true;
8748            return Ok(None);
8749        }
8750
8751        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8752        let mut command = serde_json::Map::from_iter([
8753            ("type".to_string(), json!("continue_as_new")),
8754            ("arguments".to_string(), arguments),
8755            ("queue".to_string(), json!(state.task_queue.clone())),
8756        ]);
8757        if let Some(workflow_type) = request.options.workflow_type {
8758            command.insert("workflow_type".to_string(), json!(workflow_type));
8759        }
8760        if let Some(task_queue) = request.options.task_queue {
8761            command.insert("queue".to_string(), json!(task_queue));
8762        }
8763        Ok(Some(Value::Object(command)))
8764    }
8765
8766    fn matched_recorded_pending(&self) -> Result<bool> {
8767        let state = self
8768            .state
8769            .lock()
8770            .map_err(|_| Error::WorkflowStatePoisoned)?;
8771        Ok(state.matched_recorded_pending)
8772    }
8773
8774    fn ensure_history_consumed(&self) -> Result<()> {
8775        let state = self
8776            .state
8777            .lock()
8778            .map_err(|_| Error::WorkflowStatePoisoned)?;
8779        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8780            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8781                "recorded_commands_unconsumed",
8782                Some(command.sequence()),
8783                Some(command.shape().to_string()),
8784                Some("workflow completion".to_string()),
8785                "workflow completed before consuming all recorded durable commands",
8786            )));
8787        }
8788        if let Some(sequence) = state
8789            .recorded_continue_as_new_sequence
8790            .filter(|_| !state.continue_as_new_consumed)
8791        {
8792            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8793                "recorded_continue_as_new_unconsumed",
8794                Some(sequence),
8795                Some("continue as new".to_string()),
8796                Some("workflow completion".to_string()),
8797                "workflow completed without consuming its recorded continue-as-new transition",
8798            )));
8799        }
8800        Ok(())
8801    }
8802}
8803
8804fn contiguous_message_stream_count(
8805    pending: &[MessageStreamMessage],
8806    cursor: u64,
8807    max_items: usize,
8808) -> usize {
8809    pending
8810        .iter()
8811        .take(max_items)
8812        .enumerate()
8813        .take_while(|(offset, message)| {
8814            u64::try_from(*offset)
8815                .ok()
8816                .and_then(|offset| cursor.checked_add(offset + 1))
8817                == Some(message.position)
8818        })
8819        .count()
8820}
8821
8822fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
8823    matches!(
8824        event.event_type.as_str(),
8825        "ActivityScheduled"
8826            | "TimerScheduled"
8827            | "ChildWorkflowScheduled"
8828            | "SignalWaitOpened"
8829            | "ConditionWaitOpened"
8830            | "SearchAttributesUpserted"
8831            | "SideEffectRecorded"
8832            | "VersionMarkerRecorded"
8833            | "MemoUpserted"
8834            | "WorkflowContinuedAsNew"
8835    )
8836}
8837
8838#[derive(Debug)]
8839struct WorkflowState {
8840    workflow_id: Option<String>,
8841    run_id: Option<String>,
8842    task_queue: String,
8843    payload_codec: String,
8844    history_events: Arc<Vec<HistoryEvent>>,
8845    history_budget: WorkflowHistoryBudget,
8846    cancel_requested: bool,
8847    resume_signal: Option<ResumeSignal>,
8848    recorded_commands: Vec<RecordedCommand>,
8849    selection_markers: Vec<SelectionMarker>,
8850    selection_marker_cursor: usize,
8851    cancelled_selection_members: Vec<SelectionCancellation>,
8852    recorded_continue_as_new_sequence: Option<u64>,
8853    continue_as_new_consumed: bool,
8854    command_cursor: usize,
8855    condition_wait_occurrence_counter: u64,
8856    matched_recorded_pending: bool,
8857    version_markers: HashMap<String, (i32, u64)>,
8858    workflow_command_identity: String,
8859    workflow_stream_command_counter: u64,
8860    commands: Vec<Value>,
8861    message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
8862    message_stream_cursors: HashMap<String, u64>,
8863    message_stream_waits: HashMap<String, u64>,
8864}
8865
8866impl WorkflowState {
8867    #[cfg(test)]
8868    fn new(
8869        history: Vec<HistoryEvent>,
8870        task_queue: String,
8871        payload_codec: String,
8872        resume_signal: Option<ResumeSignal>,
8873    ) -> Result<Self> {
8874        Self::new_with_identity(
8875            history,
8876            None,
8877            None,
8878            task_queue,
8879            payload_codec,
8880            resume_signal,
8881        )
8882    }
8883
8884    fn new_with_identity(
8885        history: Vec<HistoryEvent>,
8886        workflow_id: Option<String>,
8887        run_id: Option<String>,
8888        task_queue: String,
8889        payload_codec: String,
8890        resume_signal: Option<ResumeSignal>,
8891    ) -> Result<Self> {
8892        let recorded_commands = recorded_commands(
8893            &history,
8894            &payload_codec,
8895            WorkflowIdentity {
8896                workflow_id: workflow_id.clone(),
8897                run_id: run_id.clone(),
8898            },
8899        )?;
8900        let selection_markers = recorded_selection_markers(&history)?;
8901        let cancelled_selection_members = recorded_selection_cancellations(&history)?;
8902        let recorded_continue_as_new = history
8903            .iter()
8904            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
8905            .collect::<Vec<_>>();
8906        if recorded_continue_as_new.len() > 1 {
8907            return Err(invalid_recorded_history(
8908                "duplicate_continue_as_new_transition",
8909                recorded_continue_as_new
8910                    .last()
8911                    .and_then(|event| durable_event_sequence(event))
8912                    .unwrap_or(0),
8913                "one WorkflowContinuedAsNew event",
8914                &format!(
8915                    "{} WorkflowContinuedAsNew events",
8916                    recorded_continue_as_new.len()
8917                ),
8918                "workflow history records one continue-as-new transition more than once",
8919            ));
8920        }
8921        let recorded_continue_as_new_sequence = recorded_continue_as_new
8922            .first()
8923            .map(|event| {
8924                durable_event_sequence(event).ok_or_else(|| {
8925                    Error::NonDeterministicReplay(ReplayFailure::new(
8926                        "continue_as_new_sequence_missing",
8927                        None,
8928                        Some("recorded transition sequence".to_string()),
8929                        Some("missing sequence".to_string()),
8930                        "WorkflowContinuedAsNew history is missing its recorded sequence",
8931                    ))
8932                })
8933            })
8934            .transpose()?;
8935        let mut message_stream_cursors = HashMap::new();
8936        for event in &history {
8937            if !matches!(
8938                event.event_type.as_str(),
8939                "SignalReceived" | "SignalApplied"
8940            ) || event.payload.get("signal_name").and_then(Value::as_str)
8941                != Some(MESSAGE_STREAM_SIGNAL)
8942            {
8943                continue;
8944            }
8945            let arguments = decode_signal_event_arguments(event, &payload_codec)?;
8946            if arguments.len() != 1 {
8947                continue;
8948            }
8949            let envelope = arguments[0].clone().into_json()?;
8950            let Some(envelope) = envelope.as_object() else {
8951                continue;
8952            };
8953            if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
8954            {
8955                continue;
8956            }
8957            let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
8958                continue;
8959            };
8960            let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
8961            else {
8962                continue;
8963            };
8964            let cursor = message_stream_cursors
8965                .entry(stream_name.to_string())
8966                .or_insert(0);
8967            *cursor = (*cursor).max(through_position);
8968        }
8969        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
8970        let cancel_requested = history.iter().any(|event| {
8971            matches!(
8972                event.event_type.as_str(),
8973                "WorkflowCancellationRequested" | "WorkflowCancelRequested"
8974            )
8975        });
8976        Ok(Self {
8977            workflow_command_identity: String::new(),
8978            workflow_stream_command_counter: 0,
8979            workflow_id,
8980            run_id,
8981            task_queue,
8982            payload_codec,
8983            history_events: Arc::new(history),
8984            history_budget: WorkflowHistoryBudget {
8985                event_count,
8986                ..WorkflowHistoryBudget::default()
8987            },
8988            cancel_requested,
8989            resume_signal,
8990            recorded_commands,
8991            selection_markers,
8992            selection_marker_cursor: 0,
8993            cancelled_selection_members,
8994            recorded_continue_as_new_sequence,
8995            continue_as_new_consumed: false,
8996            command_cursor: 0,
8997            condition_wait_occurrence_counter: 0,
8998            matched_recorded_pending: false,
8999            version_markers: HashMap::new(),
9000            commands: Vec::new(),
9001            message_stream_messages: HashMap::new(),
9002            message_stream_cursors,
9003            message_stream_waits: HashMap::new(),
9004        })
9005    }
9006}
9007
9008enum MessageStreamDelivery {
9009    Message(MessageStreamMessage),
9010    Cursor {
9011        stream_name: String,
9012        through_position: u64,
9013    },
9014}
9015
9016fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9017    if arguments.len() != 1 {
9018        return Ok(None);
9019    }
9020    let envelope = arguments
9021        .into_iter()
9022        .next()
9023        .expect("one argument was checked");
9024    let Some(envelope) = envelope.as_object() else {
9025        return Ok(None);
9026    };
9027    let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9028        return Ok(None);
9029    };
9030    if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9031        let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9032        else {
9033            return Ok(None);
9034        };
9035        return Ok(Some(MessageStreamDelivery::Cursor {
9036            stream_name: stream_name.to_string(),
9037            through_position,
9038        }));
9039    }
9040    if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9041        return Ok(None);
9042    }
9043    let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9044        return Ok(None);
9045    };
9046    let Some(position) = envelope
9047        .get("position")
9048        .and_then(Value::as_u64)
9049        .filter(|value| *value > 0)
9050    else {
9051        return Ok(None);
9052    };
9053    let Some(payload_envelope) = envelope.get("payload_envelope") else {
9054        return Ok(None);
9055    };
9056    let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9057    else {
9058        return Ok(None);
9059    };
9060    let decoded = decode_avro_value(&payload_envelope)?;
9061    let AvroValue::Array(values) = decoded else {
9062        return Ok(None);
9063    };
9064    Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9065        stream_name: stream_name.to_string(),
9066        message_id: message_id.to_string(),
9067        position,
9068        arguments: values,
9069    })))
9070}
9071
9072#[derive(Clone, Debug)]
9073enum RecordedCommand {
9074    Activity {
9075        sequence: u64,
9076        activity_type: Option<String>,
9077        options: Option<RecordedActivityOptions>,
9078        outcome: Option<ActivityOutcome>,
9079        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9080    },
9081    Timer {
9082        sequence: u64,
9083        delay_seconds: u64,
9084        fired: bool,
9085        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9086    },
9087    ChildWorkflow {
9088        sequence: u64,
9089        workflow_type: Option<String>,
9090        outcome: Option<ChildWorkflowOutcome>,
9091        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9092    },
9093    SignalWait {
9094        sequence: u64,
9095        signal_name: String,
9096        value: Option<Vec<AvroValue>>,
9097        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9098    },
9099    ConditionWait {
9100        sequence: u64,
9101        occurrence_id: String,
9102        condition_key: Option<String>,
9103        predicate_identity: String,
9104        timeout_seconds: Option<u64>,
9105        result: Option<ConditionWaitResult>,
9106        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9107    },
9108    SearchAttributes {
9109        sequence: u64,
9110        attributes: Value,
9111        attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9112    },
9113    SideEffect {
9114        sequence: u64,
9115        value: AvroValue,
9116    },
9117    VersionMarker {
9118        sequence: u64,
9119        change_id: String,
9120        version: i32,
9121    },
9122    Memo {
9123        sequence: u64,
9124        entries: AvroValue,
9125    },
9126}
9127
9128#[derive(Clone, Debug, PartialEq, Eq)]
9129struct SelectionMarker {
9130    selection_group_id: String,
9131    selection_group_base_sequence: u64,
9132    selection_group_size: usize,
9133    member_key: SelectionKey,
9134    member_index: usize,
9135    member_base_sequence: u64,
9136    member_size: usize,
9137    operation_kind: String,
9138    operation_identity: String,
9139    outcome: String,
9140    resolution_event_id: String,
9141    resolution_event_type: String,
9142}
9143
9144#[derive(Clone, Debug, PartialEq, Eq)]
9145struct SelectionCancellation {
9146    selection_group_id: String,
9147    member_key: SelectionKey,
9148    member_index: usize,
9149    member_base_sequence: u64,
9150    member_size: usize,
9151    operation_kind: String,
9152    operation_identity: String,
9153}
9154
9155fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9156    let mut markers: Vec<SelectionMarker> = Vec::new();
9157    for event in events
9158        .iter()
9159        .filter(|event| event.event_type == "SelectionResolved")
9160    {
9161        let payload = &event.payload;
9162        let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9163        let group_size = required_selection_usize(payload, "selection_group_size")?;
9164        let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9165        let member_size = required_selection_usize(payload, "member_size")?;
9166        let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9167        let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9168            invalid_recorded_history(
9169                "selection_marker_invalid",
9170                base_sequence,
9171                "non-empty selection_group_id",
9172                &payload.to_string(),
9173                "selection winner history is missing its durable group identity",
9174            )
9175        })?;
9176        let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9177        if group_id != expected_group_id {
9178            return Err(invalid_recorded_history(
9179                "selection_marker_invalid",
9180                base_sequence,
9181                &expected_group_id,
9182                &group_id,
9183                "selection winner history contains an incompatible group identity",
9184            ));
9185        }
9186        let group_end = base_sequence
9187            .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9188            .unwrap_or(u64::MAX);
9189        let member_end = member_base_sequence
9190            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9191            .unwrap_or(u64::MAX);
9192        if member_index >= group_size
9193            || member_base_sequence < base_sequence
9194            || member_end > group_end
9195        {
9196            return Err(invalid_recorded_history(
9197                "selection_marker_invalid",
9198                base_sequence,
9199                "winner member within selection group bounds",
9200                &payload.to_string(),
9201                "selection winner history contains an invalid member range",
9202            ));
9203        }
9204        let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9205            invalid_recorded_history(
9206                "selection_marker_invalid",
9207                base_sequence,
9208                "selection operation kind",
9209                &payload.to_string(),
9210                "selection winner history is missing its operation kind",
9211            )
9212        })?;
9213        if !matches!(
9214            operation_kind.as_str(),
9215            "activity" | "child" | "timer" | "signal" | "condition" | "group"
9216        ) {
9217            return Err(invalid_recorded_history(
9218                "selection_marker_invalid",
9219                base_sequence,
9220                "activity, child, timer, signal, condition, or group",
9221                &operation_kind,
9222                "selection winner history contains an unsupported operation kind",
9223            ));
9224        }
9225        let operation_identity =
9226            payload_string(payload, "operation_identity").ok_or_else(|| {
9227                invalid_recorded_history(
9228                    "selection_marker_invalid",
9229                    base_sequence,
9230                    "non-empty operation identity",
9231                    &payload.to_string(),
9232                    "selection winner history is missing its durable operation identity",
9233                )
9234            })?;
9235        let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9236            invalid_recorded_history(
9237                "selection_marker_invalid",
9238                base_sequence,
9239                "completed or failed selection outcome",
9240                &payload.to_string(),
9241                "selection winner history is missing its outcome",
9242            )
9243        })?;
9244        if !matches!(outcome.as_str(), "completed" | "failed") {
9245            return Err(invalid_recorded_history(
9246                "selection_marker_invalid",
9247                base_sequence,
9248                "completed or failed selection outcome",
9249                &outcome,
9250                "selection winner history contains an unsupported outcome",
9251            ));
9252        }
9253        let marker = SelectionMarker {
9254            selection_group_id: group_id,
9255            selection_group_base_sequence: base_sequence,
9256            selection_group_size: group_size,
9257            member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9258            member_index,
9259            member_base_sequence,
9260            member_size,
9261            operation_kind,
9262            operation_identity,
9263            outcome,
9264            resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9265                || {
9266                    invalid_recorded_history(
9267                        "selection_marker_invalid",
9268                        base_sequence,
9269                        "durable resolution_event_id",
9270                        &payload.to_string(),
9271                        "selection winner history is missing its terminal event identity",
9272                    )
9273                },
9274            )?,
9275            resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9276                || {
9277                    invalid_recorded_history(
9278                        "selection_marker_invalid",
9279                        base_sequence,
9280                        "durable resolution_event_type",
9281                        &payload.to_string(),
9282                        "selection winner history is missing its terminal event type",
9283                    )
9284                },
9285            )?,
9286        };
9287        if let Some(existing) = markers
9288            .iter()
9289            .find(|existing| existing.selection_group_id == marker.selection_group_id)
9290        {
9291            if existing != &marker {
9292                return Err(invalid_recorded_history(
9293                    "selection_marker_conflict",
9294                    base_sequence,
9295                    &format!("one winner for {}", marker.selection_group_id),
9296                    &payload.to_string(),
9297                    "selection history records conflicting winners for one durable group",
9298                ));
9299            }
9300            continue;
9301        }
9302        markers.push(marker);
9303    }
9304    Ok(markers)
9305}
9306
9307fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9308    let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9309    for event in events
9310        .iter()
9311        .filter(|event| event.event_type == "SelectionOperationCancelled")
9312    {
9313        let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9314            invalid_recorded_history(
9315                "selection_cancellation_invalid",
9316                0,
9317                "non-empty selection_group_id",
9318                &event.payload.to_string(),
9319                "selection cancellation history is missing its group identity",
9320            )
9321        })?;
9322        let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9323        let marker = SelectionCancellation {
9324            selection_group_id: group_id,
9325            member_key: selection_key_from_value(
9326                event.payload.get("member_key"),
9327                member_base_sequence,
9328            )?,
9329            member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9330            member_base_sequence,
9331            member_size: required_selection_usize(&event.payload, "member_size")?,
9332            operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9333                invalid_recorded_history(
9334                    "selection_cancellation_invalid",
9335                    member_base_sequence,
9336                    "selection operation kind",
9337                    &event.payload.to_string(),
9338                    "selection cancellation is missing its operation kind",
9339                )
9340            })?,
9341            operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9342                || {
9343                    invalid_recorded_history(
9344                        "selection_cancellation_invalid",
9345                        member_base_sequence,
9346                        "selection operation identity",
9347                        &event.payload.to_string(),
9348                        "selection cancellation is missing its operation identity",
9349                    )
9350                },
9351            )?,
9352        };
9353        if let Some(existing) = cancelled.iter().find(|recorded| {
9354            recorded.selection_group_id == marker.selection_group_id
9355                && recorded.member_base_sequence == marker.member_base_sequence
9356        }) {
9357            if existing != &marker {
9358                return Err(invalid_recorded_history(
9359                    "selection_cancellation_conflict",
9360                    member_base_sequence,
9361                    "one stable SelectionOperationCancelled marker",
9362                    &event.payload.to_string(),
9363                    "selection cancellation history contains conflicting member metadata",
9364                ));
9365            }
9366        } else {
9367            cancelled.push(marker);
9368        }
9369    }
9370    Ok(cancelled)
9371}
9372
9373fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9374    payload
9375        .get(field)
9376        .and_then(value_as_u64)
9377        .filter(|value| *value > 0)
9378        .ok_or_else(|| {
9379            invalid_recorded_history(
9380                "selection_marker_invalid",
9381                0,
9382                &format!("positive integer {field}"),
9383                &payload.to_string(),
9384                "selection history contains invalid durable identity metadata",
9385            )
9386        })
9387}
9388
9389fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9390    required_selection_usize_allow_zero(payload, field).and_then(|value| {
9391        if value > 0 {
9392            Ok(value)
9393        } else {
9394            Err(invalid_recorded_history(
9395                "selection_marker_invalid",
9396                0,
9397                &format!("positive integer {field}"),
9398                &payload.to_string(),
9399                "selection history contains invalid durable identity metadata",
9400            ))
9401        }
9402    })
9403}
9404
9405fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9406    payload
9407        .get(field)
9408        .and_then(value_as_u64)
9409        .and_then(|value| usize::try_from(value).ok())
9410        .ok_or_else(|| {
9411            invalid_recorded_history(
9412                "selection_marker_invalid",
9413                0,
9414                &format!("non-negative integer {field}"),
9415                &payload.to_string(),
9416                "selection history contains invalid durable identity metadata",
9417            )
9418        })
9419}
9420
9421#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9422struct RecordedActivityOptions {
9423    task_queue: RecordedSnapshotValue<Option<String>>,
9424    execution_mode: RecordedSnapshotValue<Option<String>>,
9425    retry_policy: ActivityRetrySnapshot,
9426}
9427
9428#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9429enum RecordedSnapshotValue<T> {
9430    /// Older history did not persist this field, so it cannot constrain replay.
9431    Unknown,
9432    Known(T),
9433}
9434
9435impl<T: PartialEq> RecordedSnapshotValue<T> {
9436    fn matches_current(&self, current: &Self) -> bool {
9437        match self {
9438            Self::Unknown => true,
9439            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9440        }
9441    }
9442}
9443
9444#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9445struct ActivityRetrySnapshot {
9446    snapshot_version: RecordedSnapshotValue<Option<u64>>,
9447    max_attempts: RecordedSnapshotValue<Option<u64>>,
9448    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9449    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9450    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9451    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9452    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9453    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9454}
9455
9456impl ActivityRetrySnapshot {
9457    fn matches_current(&self, current: &Self) -> bool {
9458        self.snapshot_version
9459            .matches_current(&current.snapshot_version)
9460            && self.max_attempts.matches_current(&current.max_attempts)
9461            && self
9462                .backoff_seconds
9463                .matches_current(&current.backoff_seconds)
9464            && self
9465                .start_to_close_timeout
9466                .matches_current(&current.start_to_close_timeout)
9467            && self
9468                .schedule_to_start_timeout
9469                .matches_current(&current.schedule_to_start_timeout)
9470            && self
9471                .schedule_to_close_timeout
9472                .matches_current(&current.schedule_to_close_timeout)
9473            && self
9474                .heartbeat_timeout
9475                .matches_current(&current.heartbeat_timeout)
9476            && self
9477                .non_retryable_error_types
9478                .matches_current(&current.non_retryable_error_types)
9479    }
9480}
9481
9482fn recorded_optional_u64(
9483    object: Option<&serde_json::Map<String, Value>>,
9484    field: &str,
9485) -> RecordedSnapshotValue<Option<u64>> {
9486    match object.and_then(|object| object.get(field)) {
9487        None => RecordedSnapshotValue::Unknown,
9488        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9489        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9490    }
9491}
9492
9493fn recorded_optional_string(
9494    object: &serde_json::Map<String, Value>,
9495    field: &str,
9496) -> RecordedSnapshotValue<Option<String>> {
9497    match object.get(field) {
9498        None => RecordedSnapshotValue::Unknown,
9499        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9500        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9501    }
9502}
9503
9504fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9505    let policy = policy.and_then(Value::as_object);
9506    let backoff_seconds = policy
9507        .and_then(|policy| policy.get("backoff_seconds"))
9508        .and_then(Value::as_array)
9509        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9510        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9511    let mut non_retryable_error_types = Vec::new();
9512    for error_type in policy
9513        .and_then(|policy| policy.get("non_retryable_error_types"))
9514        .and_then(Value::as_array)
9515        .into_iter()
9516        .flatten()
9517        .filter_map(Value::as_str)
9518        .map(str::trim)
9519        .filter(|error_type| !error_type.is_empty())
9520    {
9521        if !non_retryable_error_types
9522            .iter()
9523            .any(|recorded| recorded == error_type)
9524        {
9525            non_retryable_error_types.push(error_type.to_string());
9526        }
9527    }
9528
9529    ActivityRetrySnapshot {
9530        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9531        max_attempts: recorded_optional_u64(policy, "max_attempts"),
9532        backoff_seconds,
9533        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9534        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9535        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9536        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9537        non_retryable_error_types: if policy
9538            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9539        {
9540            RecordedSnapshotValue::Known(non_retryable_error_types)
9541        } else {
9542            RecordedSnapshotValue::Unknown
9543        },
9544    }
9545}
9546
9547fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9548    let policy = options.retry_policy.as_ref();
9549    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9550        Some(Value::Null) => None,
9551        Some(value) => value_as_u64(value),
9552        None => Some(1),
9553    };
9554    let backoff_seconds = policy
9555        .and_then(|policy| policy.get("backoff_seconds"))
9556        .and_then(Value::as_array)
9557        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9558        .unwrap_or_default();
9559    let non_retryable_error_types = policy
9560        .and_then(|policy| policy.get("non_retryable_error_types"))
9561        .and_then(Value::as_array)
9562        .into_iter()
9563        .flatten()
9564        .filter_map(Value::as_str)
9565        .map(str::to_string)
9566        .collect();
9567
9568    ActivityRetrySnapshot {
9569        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9570        max_attempts: RecordedSnapshotValue::Known(max_attempts),
9571        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9572        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9573        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9574        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9575        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9576        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9577    }
9578}
9579
9580fn activity_options_description(options: &RecordedActivityOptions) -> String {
9581    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9582}
9583
9584impl RecordedCommand {
9585    fn sequence(&self) -> u64 {
9586        match self {
9587            Self::Activity { sequence, .. }
9588            | Self::Timer { sequence, .. }
9589            | Self::ChildWorkflow { sequence, .. }
9590            | Self::SignalWait { sequence, .. }
9591            | Self::ConditionWait { sequence, .. }
9592            | Self::SearchAttributes { sequence, .. }
9593            | Self::SideEffect { sequence, .. }
9594            | Self::VersionMarker { sequence, .. }
9595            | Self::Memo { sequence, .. } => *sequence,
9596        }
9597    }
9598
9599    fn shape(&self) -> &'static str {
9600        match self {
9601            Self::Activity { .. } => "activity",
9602            Self::Timer { .. } => "timer",
9603            Self::ChildWorkflow { .. } => "child workflow",
9604            Self::SignalWait { .. } => "signal wait",
9605            Self::ConditionWait { .. } => "condition wait",
9606            Self::SearchAttributes { .. } => "search-attribute update",
9607            Self::SideEffect { .. } => "side effect",
9608            Self::VersionMarker { .. } => "version marker",
9609            Self::Memo { .. } => "memo upsert",
9610        }
9611    }
9612}
9613
9614fn ensure_version_supported(
9615    change_id: &str,
9616    version: i32,
9617    min_supported: i32,
9618    max_supported: i32,
9619    sequence: u64,
9620) -> Result<()> {
9621    if (min_supported..=max_supported).contains(&version) {
9622        return Ok(());
9623    }
9624    Err(Error::NonDeterministicReplay(ReplayFailure::new(
9625        "version_marker_incompatible_range",
9626        (sequence != 0).then_some(sequence),
9627        Some(format!("{min_supported}..={max_supported}")),
9628        Some(format!("{change_id}:{version}")),
9629        "recorded workflow version is outside the range supported by current code",
9630    )))
9631}
9632
9633#[derive(Clone, Debug)]
9634struct ResumeSignal {
9635    signal_name: String,
9636    arguments: Vec<AvroValue>,
9637}
9638
9639const MAX_PARALLEL_OPERATIONS: usize = 1000;
9640
9641fn parallel_group_prefix(kind: &str) -> &'static str {
9642    match kind {
9643        "activity" => "parallel-activities",
9644        "child" => "parallel-children",
9645        "timer" => "parallel-timers",
9646        _ => "parallel-calls",
9647    }
9648}
9649
9650fn parallel_group_entry(
9651    base_sequence: u64,
9652    size: usize,
9653    index: usize,
9654    kind: &str,
9655) -> ParallelGroupMetadata {
9656    ParallelGroupMetadata {
9657        parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9658        parallel_group_kind: kind.to_string(),
9659        parallel_group_base_sequence: base_sequence,
9660        parallel_group_size: size,
9661        parallel_group_index: index,
9662        parallel_group_mode: None,
9663        selection_member_key: None,
9664        selection_member_index: None,
9665        selection_member_base_sequence: None,
9666        selection_member_size: None,
9667        selection_member_kind: None,
9668    }
9669}
9670
9671struct SelectionMemberMetadata {
9672    key: SelectionKey,
9673    index: usize,
9674    base_sequence: u64,
9675    size: usize,
9676    kind: String,
9677}
9678
9679fn selection_group_entry(
9680    base_sequence: u64,
9681    size: usize,
9682    index: usize,
9683    kind: &str,
9684    member: &SelectionMemberMetadata,
9685) -> ParallelGroupMetadata {
9686    ParallelGroupMetadata {
9687        parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9688        parallel_group_kind: kind.to_string(),
9689        parallel_group_base_sequence: base_sequence,
9690        parallel_group_size: size,
9691        parallel_group_index: index,
9692        parallel_group_mode: Some("select".to_string()),
9693        selection_member_key: Some(member.key.clone()),
9694        selection_member_index: Some(member.index),
9695        selection_member_base_sequence: Some(member.base_sequence),
9696        selection_member_size: Some(member.size),
9697        selection_member_kind: Some(member.kind.clone()),
9698    }
9699}
9700
9701fn apply_parallel_group_path(
9702    command: &mut serde_json::Map<String, Value>,
9703    path: &[ParallelGroupMetadata],
9704) {
9705    let Some(inner) = path.last() else {
9706        return;
9707    };
9708    command.insert(
9709        "parallel_group_id".to_string(),
9710        json!(inner.parallel_group_id),
9711    );
9712    command.insert(
9713        "parallel_group_kind".to_string(),
9714        json!(inner.parallel_group_kind),
9715    );
9716    command.insert(
9717        "parallel_group_base_sequence".to_string(),
9718        json!(inner.parallel_group_base_sequence),
9719    );
9720    command.insert(
9721        "parallel_group_size".to_string(),
9722        json!(inner.parallel_group_size),
9723    );
9724    command.insert(
9725        "parallel_group_index".to_string(),
9726        json!(inner.parallel_group_index),
9727    );
9728    if let Some(mode) = &inner.parallel_group_mode {
9729        command.insert("parallel_group_mode".to_string(), json!(mode));
9730    }
9731    if let Some(key) = &inner.selection_member_key {
9732        command.insert("selection_member_key".to_string(), json!(key));
9733    }
9734    if let Some(index) = inner.selection_member_index {
9735        command.insert("selection_member_index".to_string(), json!(index));
9736    }
9737    if let Some(base_sequence) = inner.selection_member_base_sequence {
9738        command.insert(
9739            "selection_member_base_sequence".to_string(),
9740            json!(base_sequence),
9741        );
9742    }
9743    if let Some(size) = inner.selection_member_size {
9744        command.insert("selection_member_size".to_string(), json!(size));
9745    }
9746    if let Some(kind) = &inner.selection_member_kind {
9747        command.insert("selection_member_kind".to_string(), json!(kind));
9748    }
9749    command.insert("parallel_group_path".to_string(), json!(path));
9750}
9751
9752fn ensure_parallel_path_matches(
9753    sequence: u64,
9754    recorded: Option<&[ParallelGroupMetadata]>,
9755    expected: &[ParallelGroupMetadata],
9756) -> Result<()> {
9757    match (recorded, expected.is_empty()) {
9758        (None, true) => Ok(()),
9759        (Some(recorded), false) if recorded == expected => Ok(()),
9760        (None, false) => Err(invalid_recorded_history(
9761            "parallel_group_metadata_missing",
9762            sequence,
9763            &serde_json::to_string(expected).unwrap_or_default(),
9764            "<missing>",
9765            "recorded parallel member is missing its durable group path",
9766        )),
9767        (Some(recorded), true) => Err(invalid_recorded_history(
9768            "parallel_group_shape_mismatch",
9769            sequence,
9770            "sequential command",
9771            &serde_json::to_string(recorded).unwrap_or_default(),
9772            "recorded command belonged to a parallel group but current code schedules it sequentially",
9773        )),
9774        (Some(recorded), false) => Err(invalid_recorded_history(
9775            "parallel_group_shape_mismatch",
9776            sequence,
9777            &serde_json::to_string(recorded).unwrap_or_default(),
9778            &serde_json::to_string(expected).unwrap_or_default(),
9779            "recorded parallel-group identity or path changed during replay",
9780        )),
9781    }
9782}
9783
9784#[derive(Clone, Debug)]
9785enum ParallelShape {
9786    Leaf,
9787    Group(Vec<ParallelShape>),
9788}
9789
9790struct ParallelDescriptor {
9791    operation: ParallelOperation,
9792    offset: usize,
9793    member_path: Vec<usize>,
9794    group_path: Vec<ParallelGroupMetadata>,
9795}
9796
9797fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9798    operations
9799        .iter()
9800        .map(|operation| match operation {
9801            ParallelOperation::Group(children) => parallel_leaf_count(children),
9802            _ => 1,
9803        })
9804        .sum()
9805}
9806
9807fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
9808    match operation {
9809        ParallelOperation::Activity { .. } => Some("activity"),
9810        ParallelOperation::ChildWorkflow { .. } => Some("child"),
9811        ParallelOperation::Timer(_) => Some("timer"),
9812        ParallelOperation::Signal(_) => Some("signal"),
9813        ParallelOperation::Condition { .. } => Some("condition"),
9814        ParallelOperation::Group(children) => parallel_group_kind(children),
9815    }
9816}
9817
9818fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
9819    let mut kind = None;
9820    for operation in operations {
9821        let Some(operation_kind) = parallel_operation_kind(operation) else {
9822            continue;
9823        };
9824        match kind {
9825            None => kind = Some(operation_kind),
9826            Some(current) if current == operation_kind => {}
9827            Some(_) => return Some("mixed"),
9828        }
9829    }
9830    kind
9831}
9832
9833fn validate_parallel_operations(
9834    operations: &[ParallelOperation],
9835    member_path: &mut Vec<usize>,
9836    root: bool,
9837) -> Result<()> {
9838    let leaves = parallel_leaf_count(operations);
9839    if leaves > MAX_PARALLEL_OPERATIONS {
9840        return Err(Error::InvalidParallelGroup(ParallelGroupError {
9841            reason: "fan_out_limit_exceeded",
9842            member_path: member_path.clone(),
9843            message: format!(
9844                "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
9845            ),
9846        }));
9847    }
9848    if !root && operations.is_empty() {
9849        return Err(Error::InvalidParallelGroup(ParallelGroupError {
9850            reason: "nested_group_empty",
9851            member_path: member_path.clone(),
9852            message: "a nested group must contain at least one durable leaf".to_string(),
9853        }));
9854    }
9855
9856    for (index, operation) in operations.iter().enumerate() {
9857        member_path.push(index);
9858        match operation {
9859            ParallelOperation::Activity {
9860                options, arguments, ..
9861            } => {
9862                options
9863                    .validate()
9864                    .map_err(|error| Error::InvalidActivityOptions(error))?;
9865                if let Err(error) = arguments {
9866                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
9867                        reason: "arguments_invalid",
9868                        member_path: member_path.clone(),
9869                        message: error.to_string(),
9870                    }));
9871                }
9872            }
9873            ParallelOperation::ChildWorkflow {
9874                options, arguments, ..
9875            } => {
9876                validate_parallel_child_options(options)?;
9877                if let Err(error) = arguments {
9878                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
9879                        reason: "arguments_invalid",
9880                        member_path: member_path.clone(),
9881                        message: error.to_string(),
9882                    }));
9883                }
9884            }
9885            ParallelOperation::Timer(duration)
9886                if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
9887            {
9888                return Err(Error::TimerDurationOverflow);
9889            }
9890            ParallelOperation::Timer(_) => {}
9891            ParallelOperation::Signal(signal_name) => {
9892                validate_user_signal_name(signal_name)?;
9893                if signal_name.trim().is_empty() {
9894                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
9895                        reason: "signal_name_empty",
9896                        member_path: member_path.clone(),
9897                        message: "signal wait name must not be empty".to_string(),
9898                    }));
9899                }
9900            }
9901            ParallelOperation::Condition { options, .. } => {
9902                options.validate()?;
9903            }
9904            ParallelOperation::Group(children) => {
9905                validate_parallel_operations(children, member_path, false)?;
9906            }
9907        }
9908        member_path.pop();
9909    }
9910    Ok(())
9911}
9912
9913fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
9914    if options.task_queue.trim().is_empty() {
9915        return Err(Error::InvalidChildWorkflowOptions(
9916            "task_queue must not be empty".to_string(),
9917        ));
9918    }
9919    for (name, value) in [
9920        (
9921            "execution_timeout_seconds",
9922            options.execution_timeout_seconds,
9923        ),
9924        ("run_timeout_seconds", options.run_timeout_seconds),
9925    ] {
9926        if value == Some(0) {
9927            return Err(Error::InvalidChildWorkflowOptions(format!(
9928                "{name} must be at least 1"
9929            )));
9930        }
9931    }
9932    if options
9933        .retry_policy
9934        .as_ref()
9935        .is_some_and(|policy| policy.max_attempts == Some(0))
9936    {
9937        return Err(Error::InvalidChildWorkflowOptions(
9938            "retry_policy.max_attempts must be at least 1".to_string(),
9939        ));
9940    }
9941    Ok(())
9942}
9943
9944fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
9945    ParallelShape::Group(
9946        operations
9947            .iter()
9948            .map(|operation| match operation {
9949                ParallelOperation::Group(children) => parallel_shape(children),
9950                _ => ParallelShape::Leaf,
9951            })
9952            .collect(),
9953    )
9954}
9955
9956fn parallel_descriptors(
9957    operations: Vec<ParallelOperation>,
9958    base_sequence: u64,
9959) -> Result<Vec<ParallelDescriptor>> {
9960    let size = parallel_leaf_count(&operations);
9961    let kind = parallel_group_kind(&operations).unwrap_or("activity");
9962    let mut descriptors = Vec::with_capacity(size);
9963    let mut cursor = 0;
9964
9965    for (index, operation) in operations.into_iter().enumerate() {
9966        match operation {
9967            ParallelOperation::Group(children) => {
9968                let child_base = base_sequence
9969                    .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
9970                    .ok_or(Error::TimerDurationOverflow)?;
9971                for mut descriptor in parallel_descriptors(children, child_base)? {
9972                    let outer_index = cursor + descriptor.offset;
9973                    descriptor.group_path.insert(
9974                        0,
9975                        parallel_group_entry(base_sequence, size, outer_index, kind),
9976                    );
9977                    descriptor.member_path.insert(0, index);
9978                    descriptor.offset = outer_index;
9979                    descriptors.push(descriptor);
9980                }
9981                cursor = descriptors.len();
9982            }
9983            operation => {
9984                descriptors.push(ParallelDescriptor {
9985                    operation,
9986                    offset: cursor,
9987                    member_path: vec![index],
9988                    group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
9989                });
9990                cursor += 1;
9991            }
9992        }
9993    }
9994    Ok(descriptors)
9995}
9996
9997enum ParallelLeafCall {
9998    Activity(ActivityCall),
9999    ChildWorkflow(ChildWorkflowCall),
10000    Timer(TimerCall),
10001    Signal(SignalCall),
10002    Condition(ConditionWaitCall),
10003}
10004
10005fn parallel_leaf_call(
10006    ctx: &WorkflowContext,
10007    operation: ParallelOperation,
10008    parallel_group_path: Vec<ParallelGroupMetadata>,
10009) -> ParallelLeafCall {
10010    match operation {
10011        ParallelOperation::Activity {
10012            activity_type,
10013            options,
10014            arguments,
10015        } => ParallelLeafCall::Activity(ActivityCall {
10016            ctx: ctx.clone(),
10017            activity_type,
10018            options,
10019            args: Some(arguments),
10020            scheduled: false,
10021            parallel_group_path,
10022        }),
10023        ParallelOperation::ChildWorkflow {
10024            workflow_type,
10025            options,
10026            arguments,
10027        } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10028            ctx: ctx.clone(),
10029            workflow_type,
10030            options,
10031            args: Some(arguments),
10032            scheduled: false,
10033            matched_pending: false,
10034            parallel_group_path,
10035        }),
10036        ParallelOperation::Timer(duration) => {
10037            let delay_seconds = duration
10038                .as_secs()
10039                .checked_add(u64::from(duration.subsec_nanos() > 0));
10040            ParallelLeafCall::Timer(TimerCall {
10041                ctx: ctx.clone(),
10042                delay_seconds,
10043                scheduled: false,
10044                matched_pending: false,
10045                parallel_group_path,
10046            })
10047        }
10048        ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10049            ctx: ctx.clone(),
10050            signal_name,
10051            runtime_reserved_allowed: false,
10052            opened_wait: false,
10053            matched_pending: false,
10054            parallel_group_path,
10055        }),
10056        ParallelOperation::Condition { options, predicate } => {
10057            ParallelLeafCall::Condition(ConditionWaitCall {
10058                ctx: ctx.clone(),
10059                options,
10060                predicate,
10061                occurrence_id: None,
10062                opened_wait: false,
10063                parallel_group_path,
10064            })
10065        }
10066        ParallelOperation::Group(_) => {
10067            unreachable!("parallel descriptors contain only durable leaves")
10068        }
10069    }
10070}
10071
10072impl ParallelLeafCall {
10073    fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10074        match self {
10075            Self::Activity(call) => Pin::new(call)
10076                .poll_avro_value(cx)
10077                .map_ok(ParallelAvroResult::Activity),
10078            Self::ChildWorkflow(call) => Pin::new(call)
10079                .poll_avro_value(cx)
10080                .map_ok(ParallelAvroResult::ChildWorkflow),
10081            Self::Timer(call) => Pin::new(call)
10082                .poll(cx)
10083                .map_ok(|()| ParallelAvroResult::Timer),
10084            Self::Signal(call) => Pin::new(call)
10085                .poll_avro_value(cx)
10086                .map_ok(ParallelAvroResult::Signal),
10087            Self::Condition(call) => Pin::new(call)
10088                .poll(cx)
10089                .map_ok(ParallelAvroResult::Condition),
10090        }
10091    }
10092}
10093
10094struct ParallelLeaf {
10095    call: ParallelLeafCall,
10096    member_path: Vec<usize>,
10097    group_path: Vec<ParallelGroupMetadata>,
10098    result: Option<ParallelAvroResult>,
10099}
10100
10101/// Future returned by [`WorkflowContext::parallel`].
10102pub struct ParallelCall {
10103    ctx: WorkflowContext,
10104    operations: Option<Vec<ParallelOperation>>,
10105    shape: Option<ParallelShape>,
10106    leaves: Vec<ParallelLeaf>,
10107}
10108
10109impl ParallelCall {
10110    fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10111        Self {
10112            ctx,
10113            operations: Some(operations),
10114            shape: None,
10115            leaves: Vec::new(),
10116        }
10117    }
10118
10119    fn initialize(&mut self) -> Result<()> {
10120        let operations = self.operations.take().unwrap_or_default();
10121        validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10122        self.shape = Some(parallel_shape(&operations));
10123        if operations.is_empty() {
10124            return Ok(());
10125        }
10126
10127        let base_sequence = {
10128            let state = self
10129                .ctx
10130                .state
10131                .lock()
10132                .map_err(|_| Error::WorkflowStatePoisoned)?;
10133            if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10134                recorded.sequence()
10135            } else {
10136                let last = state
10137                    .recorded_commands
10138                    .last()
10139                    .map(RecordedCommand::sequence)
10140                    .unwrap_or(0);
10141                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10142                    .and_then(|sequence| sequence.checked_add(1))
10143                    .ok_or_else(|| {
10144                        Error::InvalidParallelGroup(ParallelGroupError {
10145                            reason: "sequence_overflow",
10146                            member_path: Vec::new(),
10147                            message: "parallel group sequence identity overflowed u64".to_string(),
10148                        })
10149                    })?
10150            }
10151        };
10152
10153        self.leaves = parallel_descriptors(operations, base_sequence)?
10154            .into_iter()
10155            .map(|descriptor| {
10156                let call = parallel_leaf_call(
10157                    &self.ctx,
10158                    descriptor.operation,
10159                    descriptor.group_path.clone(),
10160                );
10161                ParallelLeaf {
10162                    call,
10163                    member_path: descriptor.member_path,
10164                    group_path: descriptor.group_path,
10165                    result: None,
10166                }
10167            })
10168            .collect();
10169        Ok(())
10170    }
10171
10172    fn poll_avro_value(
10173        mut self: Pin<&mut Self>,
10174        cx: &mut TaskContext<'_>,
10175    ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10176        if self.operations.is_some() {
10177            if let Err(error) = self.initialize() {
10178                return Poll::Ready(Err(error));
10179            }
10180        }
10181        if self.leaves.is_empty() {
10182            return Poll::Ready(Ok(Vec::new()));
10183        }
10184
10185        let mut failures = Vec::new();
10186        let mut pending = false;
10187        for (index, leaf) in self.leaves.iter_mut().enumerate() {
10188            if leaf.result.is_some() {
10189                continue;
10190            }
10191            match leaf.call.poll_avro_value(cx) {
10192                Poll::Ready(Ok(result)) => leaf.result = Some(result),
10193                Poll::Ready(Err(error)) => failures.push((index, error)),
10194                Poll::Pending => pending = true,
10195            }
10196        }
10197
10198        if !failures.is_empty() {
10199            if let Some(position) = failures
10200                .iter()
10201                .position(|(_, error)| workflow_task_integrity_error(error))
10202            {
10203                return Poll::Ready(Err(failures.remove(position).1));
10204            }
10205            failures.sort_by_key(|(index, _)| *index);
10206            let (failed_index, cause) = failures.remove(0);
10207            let failed = &self.leaves[failed_index];
10208            let completed = self
10209                .leaves
10210                .iter()
10211                .filter_map(|leaf| {
10212                    leaf.result
10213                        .clone()
10214                        .and_then(|result| result.into_json_result().ok())
10215                        .map(|result| ParallelCompletion {
10216                            member_path: leaf.member_path.clone(),
10217                            result,
10218                        })
10219                })
10220                .collect();
10221            let group_id = failed
10222                .group_path
10223                .first()
10224                .map(|entry| entry.parallel_group_id.clone())
10225                .unwrap_or_default();
10226            return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10227                group_id,
10228                member_path: failed.member_path.clone(),
10229                group_path: failed.group_path.clone(),
10230                completed,
10231                cause: Box::new(cause),
10232            })));
10233        }
10234        if pending {
10235            return Poll::Pending;
10236        }
10237
10238        let mut flat_results = self
10239            .leaves
10240            .iter_mut()
10241            .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10242            .collect::<Vec<_>>()
10243            .into_iter();
10244        let results = parallel_results_for_shape(
10245            self.shape.as_ref().expect("initialized parallel shape"),
10246            &mut flat_results,
10247        );
10248        Poll::Ready(Ok(match results {
10249            ParallelAvroResult::Group(results) => results,
10250            ParallelAvroResult::Activity(_)
10251            | ParallelAvroResult::ChildWorkflow(_)
10252            | ParallelAvroResult::Timer
10253            | ParallelAvroResult::Signal(_)
10254            | ParallelAvroResult::Condition(_) => {
10255                unreachable!("root parallel shape is a group")
10256            }
10257        }))
10258    }
10259}
10260
10261fn parallel_results_for_shape(
10262    shape: &ParallelShape,
10263    flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10264) -> ParallelAvroResult {
10265    match shape {
10266        ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10267        ParallelShape::Group(children) => ParallelAvroResult::Group(
10268            children
10269                .iter()
10270                .map(|child| parallel_results_for_shape(child, flat_results))
10271                .collect(),
10272        ),
10273    }
10274}
10275
10276impl Future for ParallelCall {
10277    type Output = Result<Vec<ParallelResult>>;
10278
10279    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10280        self.poll_avro_value(cx)
10281            .map_ok(|results| {
10282                results
10283                    .into_iter()
10284                    .map(ParallelAvroResult::into_json_result)
10285                    .collect::<Result<Vec<_>>>()
10286            })
10287            .map_ok(|result| result)
10288            .flatten_result()
10289    }
10290}
10291
10292#[derive(Clone, Debug)]
10293struct SelectionMemberPlan {
10294    key: SelectionKey,
10295    index: usize,
10296    base_sequence: u64,
10297    size: usize,
10298    kind: String,
10299    shape: ParallelShape,
10300    leaf_start: usize,
10301}
10302
10303fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10304    match operation {
10305        ParallelOperation::Activity { .. } => "activity",
10306        ParallelOperation::ChildWorkflow { .. } => "child",
10307        ParallelOperation::Timer(_) => "timer",
10308        ParallelOperation::Signal(_) => "signal",
10309        ParallelOperation::Condition { .. } => "condition",
10310        ParallelOperation::Group(_) => "group",
10311    }
10312}
10313
10314fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10315    match operation {
10316        ParallelOperation::Group(children) => parallel_shape(children),
10317        _ => ParallelShape::Leaf,
10318    }
10319}
10320
10321fn selection_descriptors(
10322    operations: Vec<(SelectionKey, ParallelOperation)>,
10323    base_sequence: u64,
10324) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10325    if operations.is_empty() {
10326        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10327            reason: "selection_empty",
10328            member_path: Vec::new(),
10329            message: "durable selection requires at least one operation".to_string(),
10330        }));
10331    }
10332    let operation_refs = operations
10333        .iter()
10334        .map(|(_, operation)| operation)
10335        .collect::<Vec<_>>();
10336    let total_size = operation_refs
10337        .iter()
10338        .map(|operation| match operation {
10339            ParallelOperation::Group(children) => parallel_leaf_count(children),
10340            _ => 1,
10341        })
10342        .sum::<usize>();
10343    if total_size > MAX_PARALLEL_OPERATIONS {
10344        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10345            reason: "fan_out_limit_exceeded",
10346            member_path: Vec::new(),
10347            message: format!(
10348                "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10349            ),
10350        }));
10351    }
10352    let group_kind = {
10353        let mut kind = None;
10354        for operation in &operation_refs {
10355            let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10356            match kind {
10357                None => kind = Some(operation_kind),
10358                Some(current) if current == operation_kind => {}
10359                Some(_) => {
10360                    kind = Some("mixed");
10361                    break;
10362                }
10363            }
10364        }
10365        kind.unwrap_or("mixed")
10366    };
10367
10368    let mut descriptors = Vec::with_capacity(total_size);
10369    let mut members = Vec::with_capacity(operations.len());
10370    let mut cursor = 0usize;
10371    let mut seen_keys: Vec<SelectionKey> = Vec::new();
10372    for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10373        if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10374            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10375                reason: "selection_key_invalid",
10376                member_path: vec![member_index],
10377                message: "selection member keys must be non-empty strings or non-negative integers"
10378                    .to_string(),
10379            }));
10380        }
10381        if seen_keys.contains(&key) {
10382            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10383                reason: "selection_key_duplicate",
10384                member_path: vec![member_index],
10385                message: format!("selection member key {key:?} is duplicated"),
10386            }));
10387        }
10388        seen_keys.push(key.clone());
10389        let member_size = match &operation {
10390            ParallelOperation::Group(children) => parallel_leaf_count(children),
10391            _ => 1,
10392        };
10393        if member_size == 0 {
10394            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10395                reason: "selection_member_empty",
10396                member_path: vec![member_index],
10397                message: "a selection member must contain at least one durable leaf".to_string(),
10398            }));
10399        }
10400        let member_base = base_sequence
10401            .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10402            .ok_or(Error::TimerDurationOverflow)?;
10403        let member_kind = selection_operation_kind(&operation).to_string();
10404        let member_shape = selection_operation_shape(&operation);
10405        let leaf_start = descriptors.len();
10406        match operation {
10407            ParallelOperation::Group(children) => {
10408                validate_parallel_operations(&children, &mut vec![member_index], false)?;
10409                for mut descriptor in parallel_descriptors(children, member_base)? {
10410                    let flat_index = cursor + descriptor.offset;
10411                    descriptor.group_path.insert(
10412                        0,
10413                        selection_group_entry(
10414                            base_sequence,
10415                            total_size,
10416                            flat_index,
10417                            group_kind,
10418                            &SelectionMemberMetadata {
10419                                key: key.clone(),
10420                                index: member_index,
10421                                base_sequence: member_base,
10422                                size: member_size,
10423                                kind: member_kind.clone(),
10424                            },
10425                        ),
10426                    );
10427                    descriptor.member_path.insert(0, member_index);
10428                    descriptor.offset = flat_index;
10429                    descriptors.push(descriptor);
10430                }
10431            }
10432            operation => {
10433                validate_parallel_operations(
10434                    std::slice::from_ref(&operation),
10435                    &mut Vec::new(),
10436                    true,
10437                )?;
10438                descriptors.push(ParallelDescriptor {
10439                    operation,
10440                    offset: cursor,
10441                    member_path: vec![member_index],
10442                    group_path: vec![selection_group_entry(
10443                        base_sequence,
10444                        total_size,
10445                        cursor,
10446                        group_kind,
10447                        &SelectionMemberMetadata {
10448                            key: key.clone(),
10449                            index: member_index,
10450                            base_sequence: member_base,
10451                            size: member_size,
10452                            kind: member_kind.clone(),
10453                        },
10454                    )],
10455                });
10456            }
10457        }
10458        members.push(SelectionMemberPlan {
10459            key,
10460            index: member_index,
10461            base_sequence: member_base,
10462            size: member_size,
10463            kind: member_kind,
10464            shape: member_shape,
10465            leaf_start,
10466        });
10467        cursor += member_size;
10468    }
10469    Ok((descriptors, members))
10470}
10471
10472struct SelectionLeaf {
10473    call: ParallelLeafCall,
10474    outcome: Option<Result<ParallelAvroResult>>,
10475}
10476
10477/// Stable reference to one member of a durable selection group.
10478#[derive(Clone)]
10479pub struct DurableOperationHandle {
10480    ctx: WorkflowContext,
10481    pub key: SelectionKey,
10482    pub index: usize,
10483    pub kind: String,
10484    pub identity: String,
10485    pub base_sequence: u64,
10486    pub size: usize,
10487    pub selection_group_id: String,
10488    shape: ParallelShape,
10489}
10490
10491impl std::fmt::Debug for DurableOperationHandle {
10492    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10493        formatter
10494            .debug_struct("DurableOperationHandle")
10495            .field("key", &self.key)
10496            .field("index", &self.index)
10497            .field("kind", &self.kind)
10498            .field("identity", &self.identity)
10499            .field("base_sequence", &self.base_sequence)
10500            .field("size", &self.size)
10501            .field("selection_group_id", &self.selection_group_id)
10502            .finish()
10503    }
10504}
10505
10506impl DurableOperationHandle {
10507    /// Await this member after another member has already won the selection.
10508    pub fn await_result(&self) -> DurableOperationAwaitCall {
10509        DurableOperationAwaitCall {
10510            handle: self.clone(),
10511        }
10512    }
10513
10514    /// Request cancellation without affecting siblings. The future resolves
10515    /// to unit; only `SelectionOperationCancelled` history proves cancellation
10516    /// beat a concurrently committed terminal result.
10517    pub fn cancel(&self) -> CancelDurableOperationCall {
10518        CancelDurableOperationCall {
10519            handle: self.clone(),
10520            emitted: false,
10521        }
10522    }
10523}
10524
10525/// The one winner committed for a durable selection group.
10526#[derive(Debug)]
10527pub struct SelectionResult {
10528    pub key: SelectionKey,
10529    pub index: usize,
10530    pub kind: String,
10531    pub identity: String,
10532    pub value: Option<ParallelResult>,
10533    pub failure: Option<Error>,
10534    pub winner: DurableOperationHandle,
10535    pub handles: Vec<DurableOperationHandle>,
10536}
10537
10538impl SelectionResult {
10539    pub fn succeeded(&self) -> bool {
10540        self.failure.is_none()
10541    }
10542
10543    pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10544        self.handles.iter().find(|handle| &handle.key == key)
10545    }
10546
10547    pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10548        self.handles
10549            .iter()
10550            .filter(|handle| handle.index != self.index)
10551            .collect()
10552    }
10553
10554    pub fn into_result(self) -> Result<ParallelResult> {
10555        match (self.value, self.failure) {
10556            (Some(value), None) => Ok(value),
10557            (_, Some(error)) => Err(error),
10558            _ => Err(Error::WorkerLoop(
10559                "selection result contained neither a value nor a failure".to_string(),
10560            )),
10561        }
10562    }
10563}
10564
10565/// Future returned by [`WorkflowContext::select`].
10566pub struct SelectCall {
10567    ctx: WorkflowContext,
10568    operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10569    members: Vec<SelectionMemberPlan>,
10570    leaves: Vec<SelectionLeaf>,
10571    group_id: Option<String>,
10572}
10573
10574impl SelectCall {
10575    fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10576        Self {
10577            ctx,
10578            operations: Some(operations),
10579            members: Vec::new(),
10580            leaves: Vec::new(),
10581            group_id: None,
10582        }
10583    }
10584
10585    fn initialize(&mut self) -> Result<()> {
10586        let operations = self.operations.take().unwrap_or_default();
10587        let base_sequence = {
10588            let state = self
10589                .ctx
10590                .state
10591                .lock()
10592                .map_err(|_| Error::WorkflowStatePoisoned)?;
10593            if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10594                marker.selection_group_base_sequence
10595            } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10596                recorded.sequence()
10597            } else {
10598                let last = state
10599                    .recorded_commands
10600                    .last()
10601                    .map(RecordedCommand::sequence)
10602                    .unwrap_or(0);
10603                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10604                    .and_then(|sequence| sequence.checked_add(1))
10605                    .ok_or(Error::TimerDurationOverflow)?
10606            }
10607        };
10608        let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10609        let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10610        self.leaves = descriptors
10611            .into_iter()
10612            .map(|descriptor| SelectionLeaf {
10613                call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10614                outcome: None,
10615            })
10616            .collect();
10617        self.members = members;
10618        self.group_id = Some(group_id);
10619        Ok(())
10620    }
10621}
10622
10623impl Future for SelectCall {
10624    type Output = Result<SelectionResult>;
10625
10626    fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10627        if self.operations.is_some() {
10628            if let Err(error) = self.initialize() {
10629                return Poll::Ready(Err(error));
10630            }
10631        }
10632
10633        for leaf in &mut self.leaves {
10634            if leaf.outcome.is_some() {
10635                continue;
10636            }
10637            if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10638                if outcome
10639                    .as_ref()
10640                    .err()
10641                    .is_some_and(workflow_task_integrity_error)
10642                {
10643                    return Poll::Ready(outcome.map(|_| unreachable!()));
10644                }
10645                leaf.outcome = Some(outcome);
10646            }
10647        }
10648
10649        let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10650        let selection_member_range = self
10651            .members
10652            .first()
10653            .map(|member| member.base_sequence)
10654            .zip(self.leaves.len().try_into().ok())
10655            .map(|(base_sequence, size): (u64, u64)| {
10656                base_sequence..base_sequence.saturating_add(size)
10657            });
10658        let marker = {
10659            let mut state = match self.ctx.state.lock() {
10660                Ok(state) => state,
10661                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10662            };
10663            let marker = state
10664                .selection_markers
10665                .get(state.selection_marker_cursor)
10666                .cloned();
10667            if marker.is_none()
10668                && all_members_terminal
10669                && selection_member_range.as_ref().is_some_and(|member_range| {
10670                    state
10671                        .recorded_commands
10672                        .iter()
10673                        .any(|command| member_range.contains(&command.sequence()))
10674                })
10675            {
10676                // Terminal member history can be committed before the server's
10677                // SelectionResolved marker becomes visible to this task. That
10678                // marker is the durable barrier the selection is still waiting
10679                // on, so an otherwise commandless replay remains legitimately
10680                // pending instead of failing as an untracked yield.
10681                state.matched_recorded_pending = true;
10682            }
10683            marker
10684        };
10685        let Some(marker) = marker else {
10686            return Poll::Pending;
10687        };
10688        if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10689            || marker.selection_group_size != self.leaves.len()
10690            || self.members.first().map(|member| member.base_sequence)
10691                != Some(marker.selection_group_base_sequence)
10692        {
10693            return Poll::Ready(Err(invalid_recorded_history(
10694                "selection_group_shape_mismatch",
10695                marker.selection_group_base_sequence,
10696                self.group_id
10697                    .as_deref()
10698                    .unwrap_or("initialized selection group"),
10699                &marker.selection_group_id,
10700                "recorded selection group differs from current workflow code",
10701            )));
10702        }
10703        let Some(member_position) = self.members.iter().position(|member| {
10704            member.key == marker.member_key
10705                && member.index == marker.member_index
10706                && member.base_sequence == marker.member_base_sequence
10707                && member.size == marker.member_size
10708                && member.kind == marker.operation_kind
10709        }) else {
10710            return Poll::Ready(Err(invalid_recorded_history(
10711                "selection_member_shape_mismatch",
10712                marker.member_base_sequence,
10713                "winner member matching current workflow code",
10714                &format!("{:?}", marker.member_key),
10715                "recorded selection winner differs from the authored member identity",
10716            )));
10717        };
10718        let member = self.members[member_position].clone();
10719        let (handles, resolution_sequence) = {
10720            let mut state = match self.ctx.state.lock() {
10721                Ok(state) => state,
10722                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10723            };
10724            let identities = self
10725                .members
10726                .iter()
10727                .map(|candidate| {
10728                    selection_operation_identity(
10729                        &state,
10730                        &candidate.kind,
10731                        candidate.base_sequence,
10732                        candidate.size,
10733                    )
10734                })
10735                .collect::<Vec<_>>();
10736            if let Some((position, missing)) = identities
10737                .iter()
10738                .enumerate()
10739                .find(|(_, identity)| identity.is_empty())
10740                .map(|(position, identity)| (position, identity.clone()))
10741            {
10742                let candidate = &self.members[position];
10743                return Poll::Ready(Err(invalid_recorded_history(
10744                    "selection_operation_identity_missing",
10745                    candidate.base_sequence,
10746                    &format!(
10747                        "durable {} resource identity from scheduled/open history",
10748                        candidate.kind
10749                    ),
10750                    &missing,
10751                    "selection member history is missing its canonical durable identity",
10752                )));
10753            }
10754            let expected_winner_identity = &identities[member_position];
10755            let resolution_sequence = match validated_selection_resolution_sequence(
10756                &state,
10757                &marker,
10758                &member,
10759                expected_winner_identity,
10760            ) {
10761                Ok(sequence) => sequence,
10762                Err(error) => return Poll::Ready(Err(error)),
10763            };
10764            let handles = self
10765                .members
10766                .iter()
10767                .zip(identities)
10768                .map(|(member, identity)| DurableOperationHandle {
10769                    ctx: self.ctx.clone(),
10770                    key: member.key.clone(),
10771                    index: member.index,
10772                    kind: member.kind.clone(),
10773                    identity,
10774                    base_sequence: member.base_sequence,
10775                    size: member.size,
10776                    selection_group_id: marker.selection_group_id.clone(),
10777                    shape: member.shape.clone(),
10778                })
10779                .collect::<Vec<_>>();
10780            if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
10781                return Poll::Ready(Err(error));
10782            }
10783            state.selection_marker_cursor += 1;
10784            (handles, resolution_sequence)
10785        };
10786
10787        let mut winner_failure = None;
10788        let mut flat_results = Vec::with_capacity(member.size);
10789        if marker.outcome == "failed" {
10790            let resolution_offset = match resolution_sequence
10791                .checked_sub(member.base_sequence)
10792                .and_then(|offset| usize::try_from(offset).ok())
10793            {
10794                Some(offset) if offset < member.size => offset,
10795                _ => {
10796                    return Poll::Ready(Err(invalid_recorded_history(
10797                        "selection_resolution_event_mismatch",
10798                        member.base_sequence,
10799                        "failure event within selected member bounds",
10800                        &resolution_sequence.to_string(),
10801                        "selection failure event is outside the authored member",
10802                    )))
10803                }
10804            };
10805            let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
10806            match leaf.outcome.take() {
10807                Some(Err(error)) => winner_failure = Some(error),
10808                _ => {
10809                    return Poll::Ready(Err(invalid_recorded_history(
10810                        "selection_winner_outcome_mismatch",
10811                        member.base_sequence,
10812                        "exact failed terminal history referenced by SelectionResolved",
10813                        "missing or successful resolution event",
10814                        "selection winner marker disagrees with terminal operation history",
10815                    )))
10816                }
10817            }
10818        } else {
10819            for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
10820                match leaf.outcome.take() {
10821                    Some(Ok(result)) => flat_results.push(result),
10822                    Some(Err(_)) => {
10823                        return Poll::Ready(Err(invalid_recorded_history(
10824                            "selection_winner_outcome_mismatch",
10825                            member.base_sequence,
10826                            "fully completed nested selection member",
10827                            "failed durable leaf",
10828                            "completed selection winner contains a failed leaf",
10829                        )))
10830                    }
10831                    None => {
10832                        return Poll::Ready(Err(invalid_recorded_history(
10833                            "selection_winner_unresolved",
10834                            member.base_sequence,
10835                            "terminal history for every completed winner leaf",
10836                            "pending member history",
10837                            "completed SelectionResolved member has an unfinished durable barrier",
10838                        )))
10839                    }
10840                }
10841            }
10842        }
10843        let value = if winner_failure.is_none() {
10844            let mut flat_results = flat_results.into_iter();
10845            let value = parallel_results_for_shape(&member.shape, &mut flat_results);
10846            match value.into_json_result() {
10847                Ok(value) => Some(value),
10848                Err(error) => return Poll::Ready(Err(error)),
10849            }
10850        } else {
10851            None
10852        };
10853        let winner = handles[member_position].clone();
10854        Poll::Ready(Ok(SelectionResult {
10855            key: winner.key.clone(),
10856            index: winner.index,
10857            kind: winner.kind.clone(),
10858            identity: winner.identity.clone(),
10859            value,
10860            failure: winner_failure,
10861            winner,
10862            handles,
10863        }))
10864    }
10865}
10866
10867fn selection_operation_identity(
10868    state: &WorkflowState,
10869    kind: &str,
10870    base_sequence: u64,
10871    size: usize,
10872) -> String {
10873    if kind == "group" {
10874        return format!("group:{base_sequence}:{size}");
10875    }
10876    let fields: &[&str] = match kind {
10877        "activity" => &["activity_execution_id"],
10878        "child" => &["child_workflow_run_id"],
10879        "timer" => &["timer_id"],
10880        "signal" => &["signal_wait_id"],
10881        "condition" => &["condition_wait_id"],
10882        _ => &[],
10883    };
10884    for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
10885        for event in state
10886            .history_events
10887            .iter()
10888            .filter(|event| durable_event_sequence(event) == Some(sequence))
10889        {
10890            for field in fields {
10891                if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
10892                    if !identity.is_empty() {
10893                        return identity.to_string();
10894                    }
10895                }
10896            }
10897        }
10898    }
10899    String::new()
10900}
10901
10902fn validated_selection_resolution_sequence(
10903    state: &WorkflowState,
10904    marker: &SelectionMarker,
10905    member: &SelectionMemberPlan,
10906    expected_identity: &str,
10907) -> Result<u64> {
10908    if expected_identity.is_empty() {
10909        return Err(invalid_recorded_history(
10910            "selection_operation_identity_missing",
10911            member.base_sequence,
10912            &format!(
10913                "durable {} resource identity from scheduled/open history",
10914                member.kind
10915            ),
10916            "missing operation identity",
10917            "selection member history is missing its canonical durable identity",
10918        ));
10919    }
10920    if marker.operation_identity != expected_identity {
10921        return Err(invalid_recorded_history(
10922            "selection_operation_identity_mismatch",
10923            member.base_sequence,
10924            expected_identity,
10925            &marker.operation_identity,
10926            "selection winner identity does not match durable scheduled/open history",
10927        ));
10928    }
10929
10930    let failure_types = [
10931        "ActivityFailed",
10932        "ActivityCancelled",
10933        "ActivityTimedOut",
10934        "ChildRunFailed",
10935        "ChildRunCancelled",
10936        "ChildRunTerminated",
10937    ];
10938    let success_types = [
10939        "ActivityCompleted",
10940        "ChildRunCompleted",
10941        "TimerFired",
10942        "SignalApplied",
10943        "ConditionWaitSatisfied",
10944        "ConditionWaitTimedOut",
10945    ];
10946    let terminal_types: &[&str] = if marker.outcome == "failed" {
10947        &failure_types
10948    } else {
10949        &success_types
10950    };
10951    let mut candidates = Vec::new();
10952    for event in state.history_events.iter() {
10953        let Some(sequence) = durable_event_sequence(event) else {
10954            continue;
10955        };
10956        if sequence < member.base_sequence
10957            || sequence >= member.base_sequence.saturating_add(member.size as u64)
10958            || !terminal_types.contains(&event.event_type.as_str())
10959        {
10960            continue;
10961        }
10962        let event_id = event
10963            .raw
10964            .get("id")
10965            .or_else(|| event.raw.get("event_id"))
10966            .and_then(Value::as_str)
10967            .filter(|value| !value.is_empty())
10968            .ok_or_else(|| {
10969                invalid_recorded_history(
10970                    "selection_resolution_event_id_missing",
10971                    member.base_sequence,
10972                    "terminal selection history with a durable event id",
10973                    &event.payload.to_string(),
10974                    "selection terminal history cannot be bound to its winner marker",
10975                )
10976            })?;
10977        candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
10978    }
10979    let resolution = if marker.outcome == "failed" {
10980        candidates.first()
10981    } else {
10982        candidates.last()
10983    };
10984    let Some((event_id, event_type, sequence)) = resolution else {
10985        return Err(invalid_recorded_history(
10986            "selection_resolution_event_missing",
10987            member.base_sequence,
10988            "terminal history for the selected member",
10989            &format!("{:?}", marker.member_key),
10990            "selection winner marker has no matching durable terminal event",
10991        ));
10992    };
10993    if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
10994        return Err(invalid_recorded_history(
10995            "selection_resolution_event_mismatch",
10996            member.base_sequence,
10997            &format!("{event_type}:{event_id}"),
10998            &format!(
10999                "{}:{}",
11000                marker.resolution_event_type, marker.resolution_event_id
11001            ),
11002            "selection winner marker does not reference the event that made its member terminal",
11003        ));
11004    }
11005    Ok(*sequence)
11006}
11007
11008fn recorded_selection_member_outcome(
11009    state: &WorkflowState,
11010    handle: &DurableOperationHandle,
11011) -> Result<Option<ParallelResult>> {
11012    for event in state.history_events.iter() {
11013        let Some(sequence) = durable_event_sequence(event) else {
11014            continue;
11015        };
11016        if sequence < handle.base_sequence
11017            || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11018            || !matches!(
11019                event.event_type.as_str(),
11020                "ActivityFailed"
11021                    | "ActivityCancelled"
11022                    | "ActivityTimedOut"
11023                    | "ChildRunFailed"
11024                    | "ChildRunCancelled"
11025                    | "ChildRunTerminated"
11026            )
11027        {
11028            continue;
11029        }
11030        let Some(command) = state
11031            .recorded_commands
11032            .iter()
11033            .find(|command| command.sequence() == sequence)
11034        else {
11035            continue;
11036        };
11037        match command {
11038            RecordedCommand::Activity {
11039                outcome: Some(Err(failure)),
11040                ..
11041            } => return Err(Error::ActivityFailed(failure.clone())),
11042            RecordedCommand::ChildWorkflow {
11043                outcome: Some(Err(failure)),
11044                ..
11045            } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11046            _ => {}
11047        }
11048    }
11049
11050    let mut results = Vec::with_capacity(handle.size);
11051    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11052        let Some(command) = state
11053            .recorded_commands
11054            .iter()
11055            .find(|command| command.sequence() == sequence)
11056        else {
11057            return Ok(None);
11058        };
11059        let result = match command {
11060            RecordedCommand::Activity { outcome, .. } => match outcome {
11061                Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11062                Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11063                None => return Ok(None),
11064            },
11065            RecordedCommand::Timer { fired, .. } => {
11066                if !fired {
11067                    return Ok(None);
11068                }
11069                ParallelAvroResult::Timer
11070            }
11071            RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11072                Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11073                Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11074                None => return Ok(None),
11075            },
11076            RecordedCommand::SignalWait { value, .. } => match value {
11077                Some(value) => ParallelAvroResult::Signal(value.clone()),
11078                None => return Ok(None),
11079            },
11080            RecordedCommand::ConditionWait { result, .. } => match result {
11081                Some(result) => ParallelAvroResult::Condition(*result),
11082                None => return Ok(None),
11083            },
11084            other => {
11085                return Err(command_mismatch(
11086                    other,
11087                    format!("selected {} member", handle.kind),
11088                ))
11089            }
11090        };
11091        results.push(result);
11092    }
11093    let mut results = results.into_iter();
11094    parallel_results_for_shape(&handle.shape, &mut results)
11095        .into_json_result()
11096        .map(Some)
11097}
11098
11099fn recorded_selection_member_is_terminal(
11100    state: &WorkflowState,
11101    handle: &DurableOperationHandle,
11102) -> bool {
11103    let mut completed = 0usize;
11104    let mut all_completed = true;
11105    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11106        let Some(command) = state
11107            .recorded_commands
11108            .iter()
11109            .find(|command| command.sequence() == sequence)
11110        else {
11111            all_completed = false;
11112            continue;
11113        };
11114        let terminal = match command {
11115            RecordedCommand::Activity {
11116                outcome: Some(Err(_)),
11117                ..
11118            }
11119            | RecordedCommand::ChildWorkflow {
11120                outcome: Some(Err(_)),
11121                ..
11122            } => return true,
11123            RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11124            RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11125            RecordedCommand::Timer { fired, .. } => *fired,
11126            RecordedCommand::SignalWait { value, .. } => value.is_some(),
11127            RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11128            RecordedCommand::SearchAttributes { .. }
11129            | RecordedCommand::SideEffect { .. }
11130            | RecordedCommand::VersionMarker { .. }
11131            | RecordedCommand::Memo { .. } => false,
11132        };
11133        if !terminal {
11134            all_completed = false;
11135            continue;
11136        }
11137        completed += 1;
11138    }
11139    all_completed && completed == handle.size
11140}
11141
11142fn selection_cancellation_for_handle(
11143    state: &WorkflowState,
11144    handle: &DurableOperationHandle,
11145) -> Result<bool> {
11146    let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11147        recorded.selection_group_id == handle.selection_group_id
11148            && recorded.member_base_sequence == handle.base_sequence
11149    }) else {
11150        return Ok(false);
11151    };
11152    validate_selection_cancellation_marker(marker, handle)?;
11153    Ok(true)
11154}
11155
11156fn validate_selection_cancellations_for_handles(
11157    state: &WorkflowState,
11158    handles: &[DurableOperationHandle],
11159) -> Result<()> {
11160    let Some(group_id) = handles
11161        .first()
11162        .map(|handle| handle.selection_group_id.as_str())
11163    else {
11164        return Ok(());
11165    };
11166    for marker in state
11167        .cancelled_selection_members
11168        .iter()
11169        .filter(|marker| marker.selection_group_id == group_id)
11170    {
11171        let Some(handle) = handles
11172            .iter()
11173            .find(|handle| handle.base_sequence == marker.member_base_sequence)
11174        else {
11175            return Err(invalid_recorded_history(
11176                "selection_cancellation_member_mismatch",
11177                marker.member_base_sequence,
11178                "SelectionOperationCancelled matching an authored selection handle",
11179                &format!("{marker:?}"),
11180                "selection cancellation member base does not name an authored member",
11181            ));
11182        };
11183        validate_selection_cancellation_marker(marker, handle)?;
11184    }
11185    Ok(())
11186}
11187
11188fn validate_selection_cancellation_marker(
11189    marker: &SelectionCancellation,
11190    handle: &DurableOperationHandle,
11191) -> Result<()> {
11192    if marker.selection_group_id != handle.selection_group_id
11193        || marker.member_key != handle.key
11194        || marker.member_index != handle.index
11195        || marker.member_base_sequence != handle.base_sequence
11196        || marker.member_size != handle.size
11197        || marker.operation_kind != handle.kind
11198        || marker.operation_identity != handle.identity
11199    {
11200        return Err(invalid_recorded_history(
11201            "selection_cancellation_member_mismatch",
11202            handle.base_sequence,
11203            "SelectionOperationCancelled matching the authored selection handle",
11204            &format!("{marker:?}"),
11205            "selection cancellation history targets different authored member metadata",
11206        ));
11207    }
11208    Ok(())
11209}
11210
11211/// Future returned by [`DurableOperationHandle::await_result`].
11212pub struct DurableOperationAwaitCall {
11213    handle: DurableOperationHandle,
11214}
11215
11216impl Future for DurableOperationAwaitCall {
11217    type Output = Result<ParallelResult>;
11218
11219    fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11220        let state = match self.handle.ctx.state.lock() {
11221            Ok(state) => state,
11222            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11223        };
11224        match selection_cancellation_for_handle(&state, &self.handle) {
11225            Err(error) => return Poll::Ready(Err(error)),
11226            Ok(false) => {}
11227            Ok(true) => {
11228                return Poll::Ready(Err(Error::DurableOperationCancelled(
11229                    DurableOperationCancelled {
11230                        selection_group_id: self.handle.selection_group_id.clone(),
11231                        member_key: self.handle.key.clone(),
11232                        member_index: self.handle.index,
11233                        operation_kind: self.handle.kind.clone(),
11234                        operation_identity: self.handle.identity.clone(),
11235                    },
11236                )));
11237            }
11238        }
11239        match recorded_selection_member_outcome(&state, &self.handle) {
11240            Ok(Some(result)) => Poll::Ready(Ok(result)),
11241            Ok(None) => Poll::Pending,
11242            Err(error) => Poll::Ready(Err(error)),
11243        }
11244    }
11245}
11246
11247/// Future returned by [`DurableOperationHandle::cancel`].
11248pub struct CancelDurableOperationCall {
11249    handle: DurableOperationHandle,
11250    emitted: bool,
11251}
11252
11253impl Future for CancelDurableOperationCall {
11254    type Output = Result<()>;
11255
11256    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11257        let ctx = self.handle.ctx.clone();
11258        let mut state = match ctx.state.lock() {
11259            Ok(state) => state,
11260            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11261        };
11262        match selection_cancellation_for_handle(&state, &self.handle) {
11263            Err(error) => return Poll::Ready(Err(error)),
11264            Ok(true) => return Poll::Ready(Ok(())),
11265            Ok(false) => {}
11266        }
11267        if recorded_selection_member_is_terminal(&state, &self.handle) {
11268            return Poll::Ready(Ok(()));
11269        }
11270        if !self.emitted {
11271            state.commands.push(json!({
11272                "type": "cancel_selection_operation",
11273                "selection_group_id": self.handle.selection_group_id,
11274                "member_key": self.handle.key,
11275                "member_index": self.handle.index,
11276                "member_base_sequence": self.handle.base_sequence,
11277                "member_size": self.handle.size,
11278                "operation_kind": self.handle.kind,
11279                "operation_identity": self.handle.identity,
11280            }));
11281            self.emitted = true;
11282        }
11283        // The cancellation command is only a request. Do not expose workflow
11284        // state after cancel until committed SelectionOperationCancelled
11285        // history proves that the request won the race with member completion.
11286        Poll::Pending
11287    }
11288}
11289
11290trait PollNestedResultExt<T> {
11291    fn flatten_result(self) -> Poll<Result<T>>;
11292}
11293
11294impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11295    fn flatten_result(self) -> Poll<Result<T>> {
11296        match self {
11297            Poll::Ready(Ok(result)) => Poll::Ready(result),
11298            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11299            Poll::Pending => Poll::Pending,
11300        }
11301    }
11302}
11303
11304struct SagaCompensation {
11305    activity_type: String,
11306    options: ActivityOptions,
11307    arguments: AvroValue,
11308    registration_order: usize,
11309}
11310
11311/// Workflow-local deterministic saga compensation helper.
11312///
11313/// Register each compensation only after its forward step succeeds. Passing
11314/// the forward `Result` to [`Saga::finish`] runs compensations sequentially in
11315/// reverse registration order after any failure, including cooperative
11316/// cancellation. Each compensation is an ordinary durable activity, so replay,
11317/// duplicate delivery, and worker restart use existing history semantics.
11318pub struct Saga {
11319    ctx: WorkflowContext,
11320    compensations: Vec<SagaCompensation>,
11321}
11322
11323impl Saga {
11324    fn new(ctx: WorkflowContext) -> Self {
11325        Self {
11326            ctx,
11327            compensations: Vec::new(),
11328        }
11329    }
11330
11331    pub fn add_compensation<T: Serialize>(
11332        &mut self,
11333        activity_type: impl Into<String>,
11334        args: T,
11335    ) -> Result<&mut Self> {
11336        self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11337    }
11338
11339    pub fn add_compensation_with_options<T: Serialize>(
11340        &mut self,
11341        activity_type: impl Into<String>,
11342        options: ActivityOptions,
11343        args: T,
11344    ) -> Result<&mut Self> {
11345        let activity_type = activity_type.into();
11346        if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11347            return Err(Error::Codec(
11348                "saga compensation activity type must be non-empty without surrounding whitespace"
11349                    .to_string(),
11350            ));
11351        }
11352        options.validate().map_err(Error::InvalidActivityOptions)?;
11353        let arguments = AvroValue::from_serialize(&args)?;
11354        let registration_order = self.compensations.len() + 1;
11355        self.compensations.push(SagaCompensation {
11356            activity_type,
11357            options,
11358            arguments,
11359            registration_order,
11360        });
11361        Ok(self)
11362    }
11363
11364    /// Compensate `initiating_failure` and return the failure that must remain.
11365    pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11366        while let Some(compensation) = self.compensations.pop() {
11367            if let Err(compensation_failure) = self
11368                .ctx
11369                .activity_with_options(
11370                    compensation.activity_type.clone(),
11371                    compensation.options,
11372                    compensation.arguments,
11373                )
11374                .await
11375            {
11376                if workflow_task_integrity_error(&compensation_failure) {
11377                    return compensation_failure;
11378                }
11379                return Error::SagaCompensationFailed(SagaCompensationFailure {
11380                    initiating_failure: Box::new(initiating_failure),
11381                    compensation_failure: Box::new(compensation_failure),
11382                    compensation_activity_type: compensation.activity_type,
11383                    compensation_registration_order: compensation.registration_order,
11384                });
11385            }
11386        }
11387        initiating_failure
11388    }
11389
11390    /// Return a successful forward value or compensate and preserve its failure.
11391    pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11392        match outcome {
11393            Ok(value) => Ok(value),
11394            Err(error) => Err(self.compensate(error).await),
11395        }
11396    }
11397}
11398
11399pub struct ActivityCall {
11400    ctx: WorkflowContext,
11401    activity_type: String,
11402    options: ActivityOptions,
11403    args: Option<Result<AvroValue>>,
11404    scheduled: bool,
11405    parallel_group_path: Vec<ParallelGroupMetadata>,
11406}
11407
11408impl ActivityCall {
11409    fn poll_avro_value(
11410        mut self: Pin<&mut Self>,
11411        _cx: &mut TaskContext<'_>,
11412    ) -> Poll<Result<AvroValue>> {
11413        let ctx = self.ctx.clone();
11414        let mut state = match ctx.state.lock() {
11415            Ok(state) => state,
11416            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11417        };
11418
11419        if self.scheduled {
11420            return Poll::Pending;
11421        }
11422
11423        let options = match self.options.validate() {
11424            Ok(options) => options,
11425            Err(error) => {
11426                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11427            }
11428        };
11429        let task_queue = options
11430            .task_queue
11431            .clone()
11432            .unwrap_or_else(|| state.task_queue.clone());
11433        let current_recorded_options = RecordedActivityOptions {
11434            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11435            // Rust schedules ordinary durable activities. The server records a
11436            // non-null mode only for a specialized execution primitive.
11437            execution_mode: RecordedSnapshotValue::Known(None),
11438            retry_policy: current_activity_retry_snapshot(&options),
11439        };
11440
11441        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11442            let sequence = recorded.sequence();
11443            match recorded {
11444                RecordedCommand::Activity {
11445                    activity_type,
11446                    options: recorded_options,
11447                    outcome,
11448                    parallel_group_path,
11449                    ..
11450                } => {
11451                    if let Err(error) = ensure_parallel_path_matches(
11452                        sequence,
11453                        parallel_group_path.as_deref(),
11454                        &self.parallel_group_path,
11455                    ) {
11456                        return Poll::Ready(Err(error));
11457                    }
11458                    if let Some(recorded_type) = activity_type {
11459                        if recorded_type != self.activity_type {
11460                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11461                                ReplayFailure::new(
11462                                    "recorded_command_detail_mismatch",
11463                                    Some(sequence),
11464                                    Some(format!("activity:{recorded_type}")),
11465                                    Some(format!("activity:{}", self.activity_type)),
11466                                    "recorded activity type differs from the current workflow command",
11467                                ),
11468                            )));
11469                        }
11470                    }
11471                    if let Some(recorded_options) = recorded_options {
11472                        if !recorded_options
11473                            .task_queue
11474                            .matches_current(&current_recorded_options.task_queue)
11475                        {
11476                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11477                                ReplayFailure::new(
11478                                    "activity_task_queue_mismatch",
11479                                    Some(sequence),
11480                                    Some(activity_options_description(&recorded_options)),
11481                                    Some(activity_options_description(&current_recorded_options)),
11482                                    "recorded activity task queue differs from the current workflow command",
11483                                ),
11484                            )));
11485                        }
11486                        if !recorded_options
11487                            .execution_mode
11488                            .matches_current(&current_recorded_options.execution_mode)
11489                        {
11490                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11491                                ReplayFailure::new(
11492                                    "activity_execution_mode_mismatch",
11493                                    Some(sequence),
11494                                    Some(activity_options_description(&recorded_options)),
11495                                    Some(activity_options_description(&current_recorded_options)),
11496                                    "recorded activity execution mode differs from the current workflow command",
11497                                ),
11498                            )));
11499                        }
11500                        if !recorded_options
11501                            .retry_policy
11502                            .matches_current(&current_recorded_options.retry_policy)
11503                        {
11504                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11505                                ReplayFailure::new(
11506                                    "activity_retry_policy_mismatch",
11507                                    Some(sequence),
11508                                    Some(activity_options_description(&recorded_options)),
11509                                    Some(activity_options_description(&current_recorded_options)),
11510                                    "recorded activity retry policy differs from the current workflow command",
11511                                ),
11512                            )));
11513                        }
11514                    }
11515                    state.command_cursor += 1;
11516                    if let Some(outcome) = outcome {
11517                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11518                    }
11519                    state.matched_recorded_pending = true;
11520                    self.scheduled = true;
11521                    return Poll::Pending;
11522                }
11523                other => {
11524                    return Poll::Ready(Err(command_mismatch(
11525                        &other,
11526                        format!("activity:{}", self.activity_type),
11527                    )));
11528                }
11529            }
11530        }
11531
11532        if !self.scheduled {
11533            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11534                Ok(args) => args,
11535                Err(error) => return Poll::Ready(Err(error)),
11536            };
11537            let arguments = normalize_avro_arguments(args);
11538            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11539                Ok(envelope) => envelope,
11540                Err(error) => return Poll::Ready(Err(error)),
11541            };
11542
11543            let mut command = serde_json::Map::from_iter([
11544                ("type".to_string(), json!("schedule_activity")),
11545                (
11546                    "activity_type".to_string(),
11547                    json!(self.activity_type.clone()),
11548                ),
11549                ("queue".to_string(), json!(task_queue)),
11550                ("arguments".to_string(), envelope),
11551            ]);
11552            for (field, value) in [
11553                ("start_to_close_timeout", options.start_to_close_timeout),
11554                (
11555                    "schedule_to_start_timeout",
11556                    options.schedule_to_start_timeout,
11557                ),
11558                (
11559                    "schedule_to_close_timeout",
11560                    options.schedule_to_close_timeout,
11561                ),
11562                ("heartbeat_timeout", options.heartbeat_timeout),
11563            ] {
11564                if let Some(value) = value {
11565                    command.insert(field.to_string(), json!(value));
11566                }
11567            }
11568            if let Some(retry_policy) = options.retry_policy {
11569                command.insert("retry_policy".to_string(), retry_policy);
11570            }
11571            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11572            state.commands.push(Value::Object(command));
11573            self.scheduled = true;
11574        }
11575
11576        Poll::Pending
11577    }
11578}
11579
11580impl Future for ActivityCall {
11581    type Output = Result<Value>;
11582
11583    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11584        match self.poll_avro_value(cx) {
11585            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11586            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11587            Poll::Pending => Poll::Pending,
11588        }
11589    }
11590}
11591
11592/// Future returned by [`WorkflowContext::sleep`].
11593pub struct TimerCall {
11594    ctx: WorkflowContext,
11595    delay_seconds: Option<u64>,
11596    scheduled: bool,
11597    matched_pending: bool,
11598    parallel_group_path: Vec<ParallelGroupMetadata>,
11599}
11600
11601impl Future for TimerCall {
11602    type Output = Result<()>;
11603
11604    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11605        if self.matched_pending {
11606            return Poll::Pending;
11607        }
11608
11609        let ctx = self.ctx.clone();
11610        let Some(requested_delay) = self.delay_seconds else {
11611            return Poll::Ready(Err(Error::TimerDurationOverflow));
11612        };
11613        let mut state = match ctx.state.lock() {
11614            Ok(state) => state,
11615            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11616        };
11617
11618        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11619            match recorded {
11620                RecordedCommand::Timer {
11621                    sequence,
11622                    delay_seconds,
11623                    fired,
11624                    parallel_group_path,
11625                    ..
11626                } => {
11627                    if let Err(error) = ensure_parallel_path_matches(
11628                        sequence,
11629                        parallel_group_path.as_deref(),
11630                        &self.parallel_group_path,
11631                    ) {
11632                        return Poll::Ready(Err(error));
11633                    }
11634                    if delay_seconds != requested_delay {
11635                        return Poll::Ready(Err(Error::NonDeterministicReplay(
11636                            ReplayFailure::new(
11637                                "timer_delay_mismatch",
11638                                Some(sequence),
11639                                Some(format!("timer:{delay_seconds}s")),
11640                                Some(format!("timer:{requested_delay}s")),
11641                                "recorded timer delay differs from the current workflow command",
11642                            ),
11643                        )));
11644                    }
11645                    state.command_cursor += 1;
11646                    if fired {
11647                        return Poll::Ready(Ok(()));
11648                    }
11649                    state.matched_recorded_pending = true;
11650                    self.scheduled = true;
11651                    self.matched_pending = true;
11652                    return Poll::Pending;
11653                }
11654                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11655            }
11656        }
11657
11658        if !self.scheduled {
11659            let mut command = serde_json::Map::from_iter([
11660                ("type".to_string(), json!("start_timer")),
11661                ("delay_seconds".to_string(), json!(requested_delay)),
11662            ]);
11663            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11664            state.commands.push(Value::Object(command));
11665            self.scheduled = true;
11666        }
11667
11668        Poll::Pending
11669    }
11670}
11671
11672/// Future returned by [`WorkflowContext::wait_condition`].
11673pub struct ConditionWaitCall {
11674    ctx: WorkflowContext,
11675    options: ConditionWaitOptions,
11676    predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11677    occurrence_id: Option<String>,
11678    opened_wait: bool,
11679    parallel_group_path: Vec<ParallelGroupMetadata>,
11680}
11681
11682impl Future for ConditionWaitCall {
11683    type Output = Result<ConditionWaitResult>;
11684
11685    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11686        if self.opened_wait {
11687            return Poll::Pending;
11688        }
11689
11690        let options = match self.options.validate() {
11691            Ok(options) => options,
11692            Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11693        };
11694        let ctx = self.ctx.clone();
11695        let occurrence_id = match self.occurrence_id.as_ref() {
11696            Some(occurrence_id) => occurrence_id.clone(),
11697            None => {
11698                let mut state = match ctx.state.lock() {
11699                    Ok(state) => state,
11700                    Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11701                };
11702                let ordinal = state.condition_wait_occurrence_counter;
11703                state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11704                    Some(next) => next,
11705                    None => {
11706                        return Poll::Ready(Err(Error::WorkerLoop(
11707                            "condition wait occurrence counter overflowed".to_string(),
11708                        )))
11709                    }
11710                };
11711                let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11712                drop(state);
11713                self.occurrence_id = Some(occurrence_id.clone());
11714                occurrence_id
11715            }
11716        };
11717
11718        let recorded_result = {
11719            let mut state = match ctx.state.lock() {
11720                Ok(state) => state,
11721                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11722            };
11723            let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11724                drop(state);
11725                return self.poll_new_condition(options);
11726            };
11727            if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11728                return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11729            }
11730
11731            let mut cursor = state.command_cursor;
11732            let mut result = None;
11733            loop {
11734                let Some(RecordedCommand::ConditionWait {
11735                    sequence,
11736                    occurrence_id: recorded_occurrence_id,
11737                    condition_key,
11738                    predicate_identity,
11739                    timeout_seconds,
11740                    result: recorded_result,
11741                    parallel_group_path,
11742                    ..
11743                }) = state.recorded_commands.get(cursor)
11744                else {
11745                    break;
11746                };
11747
11748                if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
11749                    break;
11750                }
11751                if let Err(error) = ensure_parallel_path_matches(
11752                    *sequence,
11753                    parallel_group_path.as_deref(),
11754                    &self.parallel_group_path,
11755                ) {
11756                    return Poll::Ready(Err(error));
11757                }
11758                if let Err(error) = validate_recorded_condition_wait(
11759                    *sequence,
11760                    recorded_occurrence_id,
11761                    condition_key.as_deref(),
11762                    predicate_identity,
11763                    *timeout_seconds,
11764                    &occurrence_id,
11765                    &options,
11766                ) {
11767                    return Poll::Ready(Err(error));
11768                }
11769                if result == Some(ConditionWaitResult::TimedOut) {
11770                    return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
11771                        "condition_wait_reopened_after_timeout",
11772                        Some(*sequence),
11773                        Some("timed-out condition is terminal".to_string()),
11774                        Some("another physical wait-open".to_string()),
11775                        "condition history reopened one logical wait after its durable timeout",
11776                    ))));
11777                }
11778                result = *recorded_result;
11779                cursor += 1;
11780            }
11781            state.command_cursor = cursor;
11782            result
11783        };
11784
11785        if let Some(result) = recorded_result {
11786            return Poll::Ready(Ok(result));
11787        }
11788
11789        self.poll_open_condition(options)
11790    }
11791}
11792
11793impl ConditionWaitCall {
11794    fn poll_new_condition(
11795        self: Pin<&mut Self>,
11796        options: ValidatedConditionWaitOptions,
11797    ) -> Poll<Result<ConditionWaitResult>> {
11798        self.poll_open_condition(options)
11799    }
11800
11801    fn poll_open_condition(
11802        mut self: Pin<&mut Self>,
11803        options: ValidatedConditionWaitOptions,
11804    ) -> Poll<Result<ConditionWaitResult>> {
11805        let selection_member = self
11806            .parallel_group_path
11807            .first()
11808            .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
11809        match (self.predicate)() {
11810            Ok(true) if !selection_member => {
11811                return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
11812            }
11813            Ok(_) => {}
11814            Err(error) => return Poll::Ready(Err(error)),
11815        }
11816        if options.timeout_seconds == Some(0) && !selection_member {
11817            return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
11818        }
11819
11820        let ctx = self.ctx.clone();
11821        let mut state = match ctx.state.lock() {
11822            Ok(state) => state,
11823            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11824        };
11825        let mut command = serde_json::Map::from_iter([
11826            ("type".to_string(), json!("open_condition_wait")),
11827            (
11828                "condition_wait_occurrence_id".to_string(),
11829                json!(self.occurrence_id.as_deref().unwrap_or_default()),
11830            ),
11831            ("condition_key".to_string(), json!(options.condition_key)),
11832            (
11833                "condition_definition_fingerprint".to_string(),
11834                json!(options.predicate_identity),
11835            ),
11836        ]);
11837        if let Some(timeout_seconds) = options.timeout_seconds {
11838            command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
11839        }
11840        apply_parallel_group_path(&mut command, &self.parallel_group_path);
11841        state.commands.push(Value::Object(command));
11842        drop(state);
11843        self.opened_wait = true;
11844        Poll::Pending
11845    }
11846}
11847
11848fn validate_recorded_condition_wait(
11849    sequence: u64,
11850    recorded_occurrence_id: &str,
11851    recorded_key: Option<&str>,
11852    recorded_predicate_identity: &str,
11853    recorded_timeout_seconds: Option<u64>,
11854    current_occurrence_id: &str,
11855    current: &ValidatedConditionWaitOptions,
11856) -> Result<()> {
11857    if recorded_occurrence_id != current_occurrence_id {
11858        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11859            "condition_wait_occurrence_mismatch",
11860            Some(sequence),
11861            Some(recorded_occurrence_id.to_string()),
11862            Some(current_occurrence_id.to_string()),
11863            "recorded condition occurrence differs from the current authored wait position",
11864        )));
11865    }
11866    if recorded_key != Some(current.condition_key.as_str()) {
11867        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11868            "condition_wait_key_mismatch",
11869            Some(sequence),
11870            recorded_key.map(str::to_string),
11871            Some(current.condition_key.clone()),
11872            "recorded condition identity differs from the current workflow wait",
11873        )));
11874    }
11875    if recorded_predicate_identity != current.predicate_identity {
11876        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11877            "condition_wait_predicate_mismatch",
11878            Some(sequence),
11879            Some(recorded_predicate_identity.to_string()),
11880            Some(current.predicate_identity.clone()),
11881            "recorded condition predicate behavior differs from current workflow code",
11882        )));
11883    }
11884    if recorded_timeout_seconds != current.timeout_seconds {
11885        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11886            "condition_wait_timeout_mismatch",
11887            Some(sequence),
11888            recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
11889            current.timeout_seconds.map(|seconds| format!("{seconds}s")),
11890            "recorded condition timeout differs from the current workflow wait",
11891        )));
11892    }
11893    Ok(())
11894}
11895
11896/// Future returned by [`WorkflowContext::start_child_workflow`].
11897pub struct ChildWorkflowCall {
11898    ctx: WorkflowContext,
11899    workflow_type: String,
11900    options: ChildWorkflowOptions,
11901    args: Option<Result<AvroValue>>,
11902    scheduled: bool,
11903    matched_pending: bool,
11904    parallel_group_path: Vec<ParallelGroupMetadata>,
11905}
11906
11907impl ChildWorkflowCall {
11908    fn poll_avro_value(
11909        mut self: Pin<&mut Self>,
11910        _cx: &mut TaskContext<'_>,
11911    ) -> Poll<Result<ChildWorkflowAvroResult>> {
11912        if self.matched_pending {
11913            return Poll::Pending;
11914        }
11915
11916        let ctx = self.ctx.clone();
11917        let mut state = match ctx.state.lock() {
11918            Ok(state) => state,
11919            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11920        };
11921
11922        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11923            let sequence = recorded.sequence();
11924            match recorded {
11925                RecordedCommand::ChildWorkflow {
11926                    workflow_type,
11927                    outcome,
11928                    parallel_group_path,
11929                    ..
11930                } => {
11931                    if let Err(error) = ensure_parallel_path_matches(
11932                        sequence,
11933                        parallel_group_path.as_deref(),
11934                        &self.parallel_group_path,
11935                    ) {
11936                        return Poll::Ready(Err(error));
11937                    }
11938                    if let Some(recorded_type) = workflow_type {
11939                        if recorded_type != self.workflow_type {
11940                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11941                                ReplayFailure::new(
11942                                    "recorded_command_detail_mismatch",
11943                                    Some(sequence),
11944                                    Some(format!("child workflow:{recorded_type}")),
11945                                    Some(format!("child workflow:{}", self.workflow_type)),
11946                                    "recorded child workflow type differs from the current workflow command",
11947                                ),
11948                            )));
11949                        }
11950                    }
11951                    state.command_cursor += 1;
11952                    if let Some(outcome) = outcome {
11953                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
11954                    }
11955                    state.matched_recorded_pending = true;
11956                    self.scheduled = true;
11957                    self.matched_pending = true;
11958                    return Poll::Pending;
11959                }
11960                other => {
11961                    return Poll::Ready(Err(command_mismatch(
11962                        &other,
11963                        format!("child workflow:{}", self.workflow_type),
11964                    )));
11965                }
11966            }
11967        }
11968
11969        if !self.scheduled {
11970            if self.options.task_queue.trim().is_empty() {
11971                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
11972                    "task_queue must not be empty".to_string(),
11973                )));
11974            }
11975            for (name, value) in [
11976                (
11977                    "execution_timeout_seconds",
11978                    self.options.execution_timeout_seconds,
11979                ),
11980                ("run_timeout_seconds", self.options.run_timeout_seconds),
11981            ] {
11982                if value == Some(0) {
11983                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
11984                        "{name} must be at least 1"
11985                    ))));
11986                }
11987            }
11988
11989            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11990                Ok(args) => args,
11991                Err(error) => return Poll::Ready(Err(error)),
11992            };
11993            let arguments = match encode_typed_envelope(
11994                &normalize_avro_arguments(args),
11995                &state.payload_codec,
11996            ) {
11997                Ok(arguments) => arguments,
11998                Err(error) => return Poll::Ready(Err(error)),
11999            };
12000            let mut command = json!({
12001                "type": "start_child_workflow",
12002                "workflow_type": self.workflow_type,
12003                "queue": self.options.task_queue,
12004                "parent_close_policy": self.options.parent_close_policy.as_str(),
12005                "arguments": arguments,
12006            });
12007            let object = command
12008                .as_object_mut()
12009                .expect("child workflow command is always an object");
12010            if let Some(policy) = &self.options.retry_policy {
12011                let mut retry_policy = serde_json::Map::new();
12012                if let Some(max_attempts) = policy.max_attempts {
12013                    if max_attempts == 0 {
12014                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12015                            "retry_policy.max_attempts must be at least 1".to_string(),
12016                        )));
12017                    }
12018                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12019                }
12020                if !policy.backoff_seconds.is_empty() {
12021                    retry_policy
12022                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12023                }
12024                if !policy.non_retryable_error_types.is_empty() {
12025                    retry_policy.insert(
12026                        "non_retryable_error_types".to_string(),
12027                        json!(policy.non_retryable_error_types),
12028                    );
12029                }
12030                if retry_policy.is_empty() {
12031                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12032                        "retry_policy must configure at least one field".to_string(),
12033                    )));
12034                }
12035                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12036            }
12037            if let Some(seconds) = self.options.execution_timeout_seconds {
12038                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12039            }
12040            if let Some(seconds) = self.options.run_timeout_seconds {
12041                object.insert("run_timeout_seconds".to_string(), json!(seconds));
12042            }
12043            apply_parallel_group_path(object, &self.parallel_group_path);
12044            state.commands.push(command);
12045            self.scheduled = true;
12046        }
12047
12048        Poll::Pending
12049    }
12050}
12051
12052impl Future for ChildWorkflowCall {
12053    type Output = Result<ChildWorkflowResult>;
12054
12055    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12056        match self.poll_avro_value(cx) {
12057            Poll::Ready(Ok(result)) => match result.result.into_json() {
12058                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12059                    parent: result.parent,
12060                    child: result.child,
12061                    child_workflow_type: result.child_workflow_type,
12062                    result: projected,
12063                })),
12064                Err(error) => Poll::Ready(Err(error)),
12065            },
12066            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12067            Poll::Pending => Poll::Pending,
12068        }
12069    }
12070}
12071
12072fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12073    Error::NonDeterministicReplay(ReplayFailure::new(
12074        "recorded_command_mismatch",
12075        Some(recorded.sequence()),
12076        Some(recorded.shape().to_string()),
12077        Some(actual.into()),
12078        "current workflow command does not match the recorded durable command sequence",
12079    ))
12080}
12081
12082pub struct SignalCall {
12083    ctx: WorkflowContext,
12084    signal_name: String,
12085    runtime_reserved_allowed: bool,
12086    opened_wait: bool,
12087    matched_pending: bool,
12088    parallel_group_path: Vec<ParallelGroupMetadata>,
12089}
12090
12091impl SignalCall {
12092    fn poll_avro_value(
12093        mut self: Pin<&mut Self>,
12094        _cx: &mut TaskContext<'_>,
12095    ) -> Poll<Result<Vec<AvroValue>>> {
12096        if self.matched_pending {
12097            return Poll::Pending;
12098        }
12099        if !self.runtime_reserved_allowed {
12100            if let Err(error) = validate_user_signal_name(&self.signal_name) {
12101                return Poll::Ready(Err(error));
12102            }
12103        }
12104
12105        let ctx = self.ctx.clone();
12106        let mut state = match ctx.state.lock() {
12107            Ok(state) => state,
12108            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12109        };
12110
12111        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12112            match recorded {
12113                RecordedCommand::SignalWait {
12114                    sequence,
12115                    signal_name,
12116                    value,
12117                    parallel_group_path,
12118                } => {
12119                    if let Err(error) = ensure_parallel_path_matches(
12120                        sequence,
12121                        parallel_group_path.as_deref(),
12122                        &self.parallel_group_path,
12123                    ) {
12124                        return Poll::Ready(Err(error));
12125                    }
12126                    if signal_name != self.signal_name {
12127                        return Poll::Ready(Err(Error::NonDeterministicReplay(
12128                            ReplayFailure::new(
12129                                "recorded_command_detail_mismatch",
12130                                Some(sequence),
12131                                Some(format!("signal wait:{signal_name}")),
12132                                Some(format!("signal wait:{}", self.signal_name)),
12133                                "recorded signal name differs from the current workflow command",
12134                            ),
12135                        )));
12136                    }
12137
12138                    state.command_cursor += 1;
12139                    if let Some(value) = value {
12140                        return Poll::Ready(Ok(value));
12141                    }
12142                    if state
12143                        .resume_signal
12144                        .as_ref()
12145                        .is_some_and(|signal| signal.signal_name == self.signal_name)
12146                    {
12147                        let signal = state
12148                            .resume_signal
12149                            .take()
12150                            .expect("matching resume signal is present");
12151                        return Poll::Ready(Ok(signal.arguments));
12152                    }
12153
12154                    state.matched_recorded_pending = true;
12155                    self.opened_wait = true;
12156                    self.matched_pending = true;
12157                    return Poll::Pending;
12158                }
12159                other => {
12160                    return Poll::Ready(Err(command_mismatch(
12161                        &other,
12162                        format!("signal wait:{}", self.signal_name),
12163                    )));
12164                }
12165            }
12166        }
12167
12168        if state
12169            .resume_signal
12170            .as_ref()
12171            .is_some_and(|signal| signal.signal_name == self.signal_name)
12172        {
12173            let signal = state
12174                .resume_signal
12175                .take()
12176                .expect("matching resume signal is present");
12177            return Poll::Ready(Ok(signal.arguments));
12178        }
12179
12180        if !self.opened_wait {
12181            let mut command = serde_json::Map::from_iter([
12182                ("type".to_string(), json!("open_signal_wait")),
12183                ("signal_name".to_string(), json!(self.signal_name)),
12184            ]);
12185            apply_parallel_group_path(&mut command, &self.parallel_group_path);
12186            state.commands.push(Value::Object(command));
12187            self.opened_wait = true;
12188        }
12189
12190        Poll::Pending
12191    }
12192}
12193
12194impl Future for SignalCall {
12195    type Output = Result<Vec<Value>>;
12196
12197    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12198        match self.poll_avro_value(cx) {
12199            Poll::Ready(Ok(values)) => Poll::Ready(
12200                values
12201                    .into_iter()
12202                    .map(AvroValue::into_json)
12203                    .collect::<Result<Vec<_>>>(),
12204            ),
12205            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12206            Poll::Pending => Poll::Pending,
12207        }
12208    }
12209}
12210
12211#[derive(Clone, Debug)]
12212pub struct ActivityContext {
12213    client: Client,
12214    pub task_id: String,
12215    pub activity_attempt_id: String,
12216    pub lease_owner: String,
12217    pub activity_type: String,
12218    pub attempt_number: u64,
12219    pub task_queue: String,
12220    pub worker_id: String,
12221}
12222
12223impl ActivityContext {
12224    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12225        self.client
12226            .heartbeat_activity_task(
12227                &self.task_id,
12228                &self.activity_attempt_id,
12229                &self.lease_owner,
12230                details,
12231            )
12232            .await
12233    }
12234}
12235
12236fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12237    validate_payload_codec(codec)?;
12238    match value {
12239        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12240            value, codec,
12241        )?)),
12242        None => Ok(AvroValue::Array(Vec::new())),
12243    }
12244}
12245
12246fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12247    let Some(signal_name) = task
12248        .signal_name
12249        .as_deref()
12250        .filter(|value| !value.is_empty())
12251    else {
12252        return Ok(None);
12253    };
12254    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12255    let AvroValue::Array(arguments) = decoded else {
12256        unreachable!("normalize_avro_arguments always returns an array");
12257    };
12258
12259    Ok(Some(ResumeSignal {
12260        signal_name: signal_name.to_string(),
12261        arguments,
12262    }))
12263}
12264
12265fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12266    validate_payload_codec(&task.payload_codec)?;
12267    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12268    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12269    for event in &task.history_events {
12270        validate_history_event_payloads(event, &task.payload_codec)?;
12271    }
12272    Ok(())
12273}
12274
12275fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12276    validate_payload_codec(&task.payload_codec)?;
12277    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12278}
12279
12280fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12281    validate_payload_codec(&task.payload_codec)?;
12282    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12283    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12284    for event in &task.history_events {
12285        validate_history_event_payloads(event, &task.payload_codec)?;
12286    }
12287
12288    let Some(export) = task.history_export.as_ref() else {
12289        return Ok(());
12290    };
12291    let export_codec = match export.get("payloads") {
12292        Some(payloads) => declared_payload_codec(payloads, "codec")?,
12293        None => None,
12294    }
12295    .unwrap_or(&task.payload_codec);
12296    validate_payload_codec(export_codec)?;
12297
12298    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12299        for event in events {
12300            let event_type = event
12301                .get("event_type")
12302                .or_else(|| event.get("type"))
12303                .and_then(Value::as_str)
12304                .unwrap_or_default();
12305            if let Some(payload) = event.get("payload") {
12306                validate_history_payloads(event_type, payload, export_codec)?;
12307            }
12308        }
12309    }
12310    for signal in export
12311        .get("signals")
12312        .and_then(Value::as_array)
12313        .into_iter()
12314        .flatten()
12315    {
12316        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12317        validate_payload_codec(codec)?;
12318        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12319    }
12320    for activity in export
12321        .get("activities")
12322        .and_then(Value::as_array)
12323        .into_iter()
12324        .flatten()
12325    {
12326        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12327        validate_payload_codec(codec)?;
12328        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12329        validate_optional_inbound_payload(activity.get("result"), codec)?;
12330    }
12331    Ok(())
12332}
12333
12334fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12335    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12336}
12337
12338fn validate_history_payloads(
12339    event_type: &str,
12340    payload: &Value,
12341    fallback_codec: &str,
12342) -> Result<()> {
12343    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12344    validate_payload_codec(codec)?;
12345    for field in history_payload_fields(event_type) {
12346        validate_optional_inbound_payload(payload.get(*field), codec)?;
12347    }
12348    Ok(())
12349}
12350
12351const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12352
12353fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12354    match event_type {
12355        "ActivityCompleted" => &["result"],
12356        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12357        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12358        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12359        "ChildRunCompleted" => &["result", "output"],
12360        "WorkflowCompleted" => &["output"],
12361        "ServiceCallStarted"
12362        | "ServiceCallCompleted"
12363        | "ServiceCallFailed"
12364        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12365        _ => &[],
12366    }
12367}
12368
12369fn signal_history_payload(payload: &Value) -> Option<&Value> {
12370    SIGNAL_HISTORY_PAYLOAD_FIELDS
12371        .iter()
12372        .find_map(|field| payload.get(*field))
12373}
12374
12375fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12376    match value.get(field) {
12377        None => Ok(None),
12378        Some(Value::String(codec)) => Ok(Some(codec)),
12379        Some(_) => Err(invalid_payload_envelope()),
12380    }
12381}
12382
12383fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12384    validate_payload_codec(codec)?;
12385    if let Some(value) = value.filter(|value| !value.is_null()) {
12386        decode_wire_avro_value(value, codec)?;
12387    }
12388    Ok(())
12389}
12390
12391fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12392    let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12393        invalid_recorded_history(
12394            "parallel_group_metadata_invalid",
12395            sequence,
12396            "non-empty parallel_group_id",
12397            &payload.to_string(),
12398            "parallel-group history is missing its stable identity",
12399        )
12400    })?;
12401    let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12402        invalid_recorded_history(
12403            "parallel_group_metadata_invalid",
12404            sequence,
12405            "activity, child, timer, signal, condition, or mixed group kind",
12406            &payload.to_string(),
12407            "parallel-group history is missing its group kind",
12408        )
12409    })?;
12410    if !matches!(
12411        kind.as_str(),
12412        "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12413    ) {
12414        return Err(invalid_recorded_history(
12415            "parallel_group_metadata_invalid",
12416            sequence,
12417            "activity, child, timer, signal, condition, or mixed group kind",
12418            &kind,
12419            "parallel-group history contains an unsupported group kind",
12420        ));
12421    }
12422    let base_sequence = payload
12423        .get("parallel_group_base_sequence")
12424        .and_then(value_as_u64)
12425        .filter(|value| *value > 0)
12426        .ok_or_else(|| {
12427            invalid_recorded_history(
12428                "parallel_group_metadata_invalid",
12429                sequence,
12430                "positive parallel_group_base_sequence",
12431                &payload.to_string(),
12432                "parallel-group history contains an invalid base sequence",
12433            )
12434        })?;
12435    let size = payload
12436        .get("parallel_group_size")
12437        .and_then(value_as_u64)
12438        .and_then(|value| usize::try_from(value).ok())
12439        .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12440        .ok_or_else(|| {
12441            invalid_recorded_history(
12442                "parallel_group_metadata_invalid",
12443                sequence,
12444                "bounded positive parallel_group_size",
12445                &payload.to_string(),
12446                "parallel-group history contains an invalid group size",
12447            )
12448        })?;
12449    let index = payload
12450        .get("parallel_group_index")
12451        .and_then(value_as_u64)
12452        .and_then(|value| usize::try_from(value).ok())
12453        .filter(|value| *value < size)
12454        .ok_or_else(|| {
12455            invalid_recorded_history(
12456                "parallel_group_metadata_invalid",
12457                sequence,
12458                "parallel_group_index within group bounds",
12459                &payload.to_string(),
12460                "parallel-group history contains an invalid member index",
12461            )
12462        })?;
12463    if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12464        return Err(invalid_recorded_history(
12465            "parallel_group_metadata_invalid",
12466            sequence,
12467            "base sequence plus member index equals workflow sequence",
12468            &payload.to_string(),
12469            "parallel-group path does not preserve durable workflow position",
12470        ));
12471    }
12472    let mode = payload
12473        .get("parallel_group_mode")
12474        .and_then(Value::as_str)
12475        .unwrap_or("all");
12476    if !matches!(mode, "all" | "select") {
12477        return Err(invalid_recorded_history(
12478            "parallel_group_metadata_invalid",
12479            sequence,
12480            "parallel group mode all or select",
12481            mode,
12482            "parallel-group history contains an unsupported group mode",
12483        ));
12484    }
12485    let expected_id = if mode == "select" {
12486        format!("select-calls:{base_sequence}:{size}")
12487    } else {
12488        format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12489    };
12490    if group_id != expected_id {
12491        return Err(invalid_recorded_history(
12492            "parallel_group_metadata_invalid",
12493            sequence,
12494            &expected_id,
12495            &group_id,
12496            "parallel-group history contains an incompatible stable group ID",
12497        ));
12498    }
12499    let selection_member_key = if mode == "select" {
12500        Some(selection_key_from_value(
12501            payload.get("selection_member_key"),
12502            sequence,
12503        )?)
12504    } else {
12505        None
12506    };
12507    let selection_member_index = if mode == "select" {
12508        Some(required_parallel_usize(
12509            payload,
12510            "selection_member_index",
12511            sequence,
12512        )?)
12513    } else {
12514        None
12515    };
12516    let selection_member_base_sequence = if mode == "select" {
12517        Some(
12518            payload
12519                .get("selection_member_base_sequence")
12520                .and_then(value_as_u64)
12521                .filter(|value| *value >= base_sequence)
12522                .ok_or_else(|| {
12523                    invalid_recorded_history(
12524                        "parallel_group_metadata_invalid",
12525                        sequence,
12526                        "selection member base within its group",
12527                        &payload.to_string(),
12528                        "selection history contains an invalid member base sequence",
12529                    )
12530                })?,
12531        )
12532    } else {
12533        None
12534    };
12535    let selection_member_size = if mode == "select" {
12536        let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12537        if member_size == 0 {
12538            return Err(invalid_recorded_history(
12539                "parallel_group_metadata_invalid",
12540                sequence,
12541                "positive selection member size",
12542                &payload.to_string(),
12543                "selection history contains an invalid member size",
12544            ));
12545        }
12546        Some(member_size)
12547    } else {
12548        None
12549    };
12550    let selection_member_kind = if mode == "select" {
12551        let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12552            invalid_recorded_history(
12553                "parallel_group_metadata_invalid",
12554                sequence,
12555                "selection member operation kind",
12556                &payload.to_string(),
12557                "selection history is missing its authored member kind",
12558            )
12559        })?;
12560        if !matches!(
12561            kind.as_str(),
12562            "activity" | "child" | "timer" | "signal" | "condition" | "group"
12563        ) {
12564            return Err(invalid_recorded_history(
12565                "parallel_group_metadata_invalid",
12566                sequence,
12567                "activity, child, timer, signal, condition, or group selection member kind",
12568                &kind,
12569                "selection history contains an unsupported member kind",
12570            ));
12571        }
12572        Some(kind)
12573    } else {
12574        None
12575    };
12576    if let (Some(member_base), Some(member_size)) =
12577        (selection_member_base_sequence, selection_member_size)
12578    {
12579        let member_end = member_base
12580            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12581            .ok_or_else(|| {
12582                invalid_recorded_history(
12583                    "parallel_group_metadata_invalid",
12584                    sequence,
12585                    "bounded selection member range",
12586                    &payload.to_string(),
12587                    "selection member range overflowed",
12588                )
12589            })?;
12590        let group_end = base_sequence
12591            .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12592            .unwrap_or(u64::MAX);
12593        if sequence < member_base || sequence >= member_end || member_end > group_end {
12594            return Err(invalid_recorded_history(
12595                "parallel_group_metadata_invalid",
12596                sequence,
12597                "workflow sequence within one bounded selection member",
12598                &payload.to_string(),
12599                "selection member range does not contain its durable leaf",
12600            ));
12601        }
12602    }
12603    Ok(ParallelGroupMetadata {
12604        parallel_group_id: group_id,
12605        parallel_group_kind: kind,
12606        parallel_group_base_sequence: base_sequence,
12607        parallel_group_size: size,
12608        parallel_group_index: index,
12609        parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12610        selection_member_key,
12611        selection_member_index,
12612        selection_member_base_sequence,
12613        selection_member_size,
12614        selection_member_kind,
12615    })
12616}
12617
12618fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12619    payload
12620        .get(field)
12621        .and_then(value_as_u64)
12622        .and_then(|value| usize::try_from(value).ok())
12623        .ok_or_else(|| {
12624            invalid_recorded_history(
12625                "parallel_group_metadata_invalid",
12626                sequence,
12627                &format!("non-negative integer {field}"),
12628                &payload.to_string(),
12629                "selection history contains invalid member metadata",
12630            )
12631        })
12632}
12633
12634fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12635    match value {
12636        Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12637        Some(value) => value_as_u64(value)
12638            .and_then(|value| usize::try_from(value).ok())
12639            .map(SelectionKey::Index)
12640            .ok_or_else(|| {
12641                invalid_recorded_history(
12642                    "selection_member_key_invalid",
12643                    sequence,
12644                    "non-empty string or non-negative integer member key",
12645                    &value.to_string(),
12646                    "selection history contains an invalid member key",
12647                )
12648            }),
12649        None => Err(invalid_recorded_history(
12650            "selection_member_key_missing",
12651            sequence,
12652            "selection_member_key",
12653            "<missing>",
12654            "selection history is missing its stable member key",
12655        )),
12656    }
12657}
12658
12659fn recorded_parallel_group_path(
12660    events: &[&HistoryEvent],
12661    sequence: u64,
12662) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12663    let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12664    for event in events {
12665        let payload = &event.payload;
12666        let has_metadata = payload.get("parallel_group_path").is_some()
12667            || payload.get("parallel_group_id").is_some()
12668            || payload.get("parallel_group_kind").is_some()
12669            || payload.get("parallel_group_base_sequence").is_some()
12670            || payload.get("parallel_group_size").is_some()
12671            || payload.get("parallel_group_index").is_some()
12672            || payload.get("parallel_group_mode").is_some()
12673            || payload.get("selection_member_key").is_some();
12674        if !has_metadata {
12675            continue;
12676        }
12677
12678        let top_level = recorded_parallel_group_entry(payload, sequence)?;
12679        let path = match payload.get("parallel_group_path") {
12680            None => vec![top_level.clone()],
12681            Some(Value::Array(entries)) if !entries.is_empty() => entries
12682                .iter()
12683                .map(|entry| recorded_parallel_group_entry(entry, sequence))
12684                .collect::<Result<Vec<_>>>()?,
12685            Some(value) => {
12686                return Err(invalid_recorded_history(
12687                    "parallel_group_metadata_invalid",
12688                    sequence,
12689                    "non-empty parallel_group_path list",
12690                    &value.to_string(),
12691                    "parallel-group history contains an invalid group path",
12692                ));
12693            }
12694        };
12695        if path.last() != Some(&top_level) {
12696            return Err(invalid_recorded_history(
12697                "parallel_group_metadata_invalid",
12698                sequence,
12699                &serde_json::to_string(&path.last()).unwrap_or_default(),
12700                &serde_json::to_string(&top_level).unwrap_or_default(),
12701                "parallel-group top-level fields do not match the innermost path entry",
12702            ));
12703        }
12704        if recorded.as_ref().is_some_and(|existing| existing != &path) {
12705            return Err(invalid_recorded_history(
12706                "parallel_group_history_conflict",
12707                sequence,
12708                &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12709                &serde_json::to_string(&path).unwrap_or_default(),
12710                "parallel-group metadata changed between scheduling and resolution history",
12711            ));
12712        }
12713        recorded = Some(path);
12714    }
12715    Ok(recorded)
12716}
12717
12718fn recorded_commands(
12719    events: &[HistoryEvent],
12720    fallback_codec: &str,
12721    parent: WorkflowIdentity,
12722) -> Result<Vec<RecordedCommand>> {
12723    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12724    let mut last_new_sequence = None;
12725
12726    for event in events {
12727        let is_activity = matches!(
12728            event.event_type.as_str(),
12729            "ActivityScheduled"
12730                | "ActivityStarted"
12731                | "ActivityHeartbeatRecorded"
12732                | "ActivityRetryScheduled"
12733                | "ActivityCompleted"
12734                | "ActivityFailed"
12735                | "ActivityCancelled"
12736                | "ActivityTimedOut"
12737        );
12738        let is_workflow_timer = matches!(
12739            event.event_type.as_str(),
12740            "TimerScheduled" | "TimerCancelled" | "TimerFired"
12741        ) && !is_internal_timer_event(event);
12742        let is_child_workflow = matches!(
12743            event.event_type.as_str(),
12744            "ChildWorkflowScheduled"
12745                | "ChildRunCompleted"
12746                | "ChildRunFailed"
12747                | "ChildRunCancelled"
12748                | "ChildRunTerminated"
12749        );
12750        let is_signal_wait = is_recorded_signal_wait_event(event);
12751        let is_condition_wait = is_recorded_condition_wait_event(event);
12752        let is_search_attributes = event.event_type == "SearchAttributesUpserted";
12753        let is_side_effect = event.event_type == "SideEffectRecorded";
12754        let is_version_marker = event.event_type == "VersionMarkerRecorded";
12755        let is_memo = event.event_type == "MemoUpserted";
12756        if !is_activity
12757            && !is_workflow_timer
12758            && !is_child_workflow
12759            && !is_signal_wait
12760            && !is_condition_wait
12761            && !is_search_attributes
12762            && !is_side_effect
12763            && !is_version_marker
12764            && !is_memo
12765        {
12766            continue;
12767        }
12768
12769        let sequence = durable_event_sequence(event).ok_or_else(|| {
12770            Error::NonDeterministicReplay(ReplayFailure::new(
12771                "durable_command_sequence_missing",
12772                None,
12773                Some("positive workflow sequence".to_string()),
12774                Some(event.event_type.clone()),
12775                "durable command history event has no workflow sequence",
12776            ))
12777        })?;
12778        if sequence == 0 {
12779            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12780                "durable_command_sequence_invalid",
12781                Some(sequence),
12782                Some("positive workflow sequence".to_string()),
12783                Some(sequence.to_string()),
12784                "durable command history uses an invalid workflow sequence",
12785            )));
12786        }
12787        if !events_by_sequence.contains_key(&sequence) {
12788            if let Some(previous) = last_new_sequence {
12789                if sequence < previous {
12790                    return Err(invalid_recorded_history(
12791                        "durable_command_sequence_mismatch",
12792                        sequence,
12793                        &format!("workflow sequence greater than {previous}"),
12794                        &sequence.to_string(),
12795                        "durable commands are not strictly ordered by their recorded workflow sequence",
12796                    ));
12797                }
12798            }
12799            last_new_sequence = Some(sequence);
12800        }
12801        events_by_sequence.entry(sequence).or_default().push(event);
12802    }
12803
12804    let commands: Vec<RecordedCommand> = events_by_sequence
12805        .into_iter()
12806        .map(|(sequence, sequence_events)| {
12807            let activity_events: Vec<_> = sequence_events
12808                .iter()
12809                .copied()
12810                .filter(|event| event.event_type.starts_with("Activity"))
12811                .collect();
12812            let timer_events: Vec<_> = sequence_events
12813                .iter()
12814                .copied()
12815                .filter(|event| event.event_type.starts_with("Timer"))
12816                .collect();
12817            let child_events: Vec<_> = sequence_events
12818                .iter()
12819                .copied()
12820                .filter(|event| {
12821                    event.event_type == "ChildWorkflowScheduled"
12822                        || event.event_type.starts_with("ChildRun")
12823                })
12824                .collect();
12825            let signal_wait_events: Vec<_> = sequence_events
12826                .iter()
12827                .copied()
12828                .filter(|event| is_recorded_signal_wait_event(event))
12829                .collect();
12830            let condition_wait_events: Vec<_> = sequence_events
12831                .iter()
12832                .copied()
12833                .filter(|event| is_recorded_condition_wait_event(event))
12834                .collect();
12835            let search_attribute_events: Vec<_> = sequence_events
12836                .iter()
12837                .copied()
12838                .filter(|event| event.event_type == "SearchAttributesUpserted")
12839                .collect();
12840            let side_effect_events: Vec<_> = sequence_events
12841                .iter()
12842                .copied()
12843                .filter(|event| event.event_type == "SideEffectRecorded")
12844                .collect();
12845            let version_marker_events: Vec<_> = sequence_events
12846                .iter()
12847                .copied()
12848                .filter(|event| event.event_type == "VersionMarkerRecorded")
12849                .collect();
12850            let memo_events: Vec<_> = sequence_events
12851                .iter()
12852                .copied()
12853                .filter(|event| event.event_type == "MemoUpserted")
12854                .collect();
12855
12856            let command_kind_count = usize::from(!activity_events.is_empty())
12857                + usize::from(!timer_events.is_empty())
12858                + usize::from(!child_events.is_empty())
12859                + usize::from(!signal_wait_events.is_empty())
12860                + usize::from(!condition_wait_events.is_empty())
12861                + usize::from(!search_attribute_events.is_empty())
12862                + usize::from(!side_effect_events.is_empty())
12863                + usize::from(!version_marker_events.is_empty())
12864                + usize::from(!memo_events.is_empty());
12865            if command_kind_count > 1 {
12866                let actual = [
12867                    (!activity_events.is_empty()).then_some("activity"),
12868                    (!timer_events.is_empty()).then_some("timer"),
12869                    (!child_events.is_empty()).then_some("child workflow"),
12870                    (!signal_wait_events.is_empty()).then_some("signal wait"),
12871                    (!condition_wait_events.is_empty()).then_some("condition wait"),
12872                    (!search_attribute_events.is_empty()).then_some("search-attribute update"),
12873                    (!side_effect_events.is_empty()).then_some("side effect"),
12874                    (!version_marker_events.is_empty()).then_some("version marker"),
12875                    (!memo_events.is_empty()).then_some("memo upsert"),
12876                ]
12877                .into_iter()
12878                .flatten()
12879                .collect::<Vec<_>>()
12880                .join(" and ");
12881                return Err(invalid_recorded_history(
12882                    "durable_command_sequence_collision",
12883                    sequence,
12884                    "one durable command kind",
12885                    &actual,
12886                    "one workflow sequence records more than one durable command kind",
12887                ));
12888            }
12889
12890            if !activity_events.is_empty() {
12891                let parallel_group_path =
12892                    recorded_parallel_group_path(&activity_events, sequence)?;
12893                let scheduled_count = activity_events
12894                    .iter()
12895                    .filter(|event| event.event_type == "ActivityScheduled")
12896                    .count();
12897                if scheduled_count > 1 {
12898                    return Err(invalid_recorded_history(
12899                        "duplicate_activity_schedule",
12900                        sequence,
12901                        "at most one ActivityScheduled event",
12902                        "multiple ActivityScheduled events",
12903                        "activity history schedules more than one command at one workflow sequence",
12904                    ));
12905                }
12906                let activity_type = activity_events.iter().find_map(|event| {
12907                    event
12908                        .payload
12909                        .get("activity_type")
12910                        .or_else(|| event.payload.get("activity_name"))
12911                        .and_then(Value::as_str)
12912                        .map(str::to_string)
12913                });
12914                if activity_events.iter().filter_map(|event| {
12915                    event
12916                        .payload
12917                        .get("activity_type")
12918                        .or_else(|| event.payload.get("activity_name"))
12919                        .and_then(Value::as_str)
12920                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
12921                    return Err(invalid_recorded_history(
12922                        "activity_identity_mismatch",
12923                        sequence,
12924                        activity_type.as_deref().unwrap_or("one activity identity"),
12925                        "conflicting activity identities",
12926                        "activity lifecycle events at one workflow sequence disagree on identity",
12927                    ));
12928                }
12929                let terminal: Vec<_> = activity_events
12930                    .iter()
12931                    .copied()
12932                    .filter(|event| {
12933                        matches!(
12934                            event.event_type.as_str(),
12935                            "ActivityCompleted"
12936                                | "ActivityFailed"
12937                                | "ActivityCancelled"
12938                                | "ActivityTimedOut"
12939                        )
12940                    })
12941                    .collect();
12942                let duplicate_delivery = terminal.first().is_some_and(|first| {
12943                    terminal.iter().all(|event| {
12944                        event.event_type == first.event_type && event.payload == first.payload
12945                    })
12946                });
12947                if terminal.len() > 1 && !duplicate_delivery {
12948                    return Err(invalid_recorded_history(
12949                        "duplicate_activity_terminal_event",
12950                        sequence,
12951                        "at most one terminal activity event",
12952                        "multiple terminal activity events",
12953                        "activity history settles one command more than once",
12954                    ));
12955                }
12956                let outcome = terminal
12957                    .first()
12958                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
12959                    .transpose()?;
12960                let options = activity_events
12961                    .iter()
12962                    .find(|event| event.event_type == "ActivityScheduled")
12963                    .and_then(|event| event.payload.get("activity"))
12964                    .and_then(Value::as_object)
12965                    .map(|activity| RecordedActivityOptions {
12966                        task_queue: recorded_optional_string(activity, "queue"),
12967                        execution_mode: recorded_optional_string(activity, "execution_mode"),
12968                        retry_policy: recorded_activity_retry_snapshot(
12969                            activity.get("retry_policy"),
12970                        ),
12971                    });
12972                return Ok(RecordedCommand::Activity {
12973                    sequence,
12974                    activity_type,
12975                    options,
12976                    outcome,
12977                    parallel_group_path,
12978                });
12979            }
12980
12981            if !child_events.is_empty() {
12982                let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
12983                let scheduled: Vec<_> = child_events
12984                    .iter()
12985                    .copied()
12986                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
12987                    .collect();
12988                if scheduled.len() != 1 {
12989                    return Err(invalid_recorded_history(
12990                        "child_workflow_schedule_missing_or_duplicate",
12991                        sequence,
12992                        "one ChildWorkflowScheduled event",
12993                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
12994                        "child workflow replay requires exactly one recorded schedule event",
12995                    ));
12996                }
12997                let workflow_type = child_events.iter().find_map(|event| {
12998                    event
12999                        .payload
13000                        .get("child_workflow_type")
13001                        .or_else(|| event.payload.get("workflow_type"))
13002                        .and_then(Value::as_str)
13003                        .filter(|value| !value.is_empty())
13004                        .map(str::to_string)
13005                });
13006                if child_events
13007                    .iter()
13008                    .filter_map(|event| {
13009                        event
13010                            .payload
13011                            .get("child_workflow_type")
13012                            .or_else(|| event.payload.get("workflow_type"))
13013                            .and_then(Value::as_str)
13014                    })
13015                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
13016                {
13017                    return Err(invalid_recorded_history(
13018                        "child_workflow_identity_mismatch",
13019                        sequence,
13020                        workflow_type
13021                            .as_deref()
13022                            .unwrap_or("one child workflow type"),
13023                        "conflicting child workflow types",
13024                        "child workflow lifecycle events at one sequence disagree on type",
13025                    ));
13026                }
13027                let mut outcomes = child_workflow_outcomes(
13028                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13029                    fallback_codec,
13030                    parent.clone(),
13031                )?;
13032                let terminal_events = child_events
13033                    .iter()
13034                    .copied()
13035                    .filter(|event| event.event_type.starts_with("ChildRun"))
13036                    .collect::<Vec<_>>();
13037                let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13038                    terminal_events.iter().all(|event| {
13039                        event.event_type == first.event_type && event.payload == first.payload
13040                    })
13041                });
13042                if outcomes.len() > 1 && !duplicate_delivery {
13043                    return Err(invalid_recorded_history(
13044                        "duplicate_child_workflow_terminal_event",
13045                        sequence,
13046                        "at most one terminal child event",
13047                        "multiple terminal child events",
13048                        "child workflow history settles one command more than once",
13049                    ));
13050                }
13051                return Ok(RecordedCommand::ChildWorkflow {
13052                    sequence,
13053                    workflow_type,
13054                    outcome: outcomes.pop(),
13055                    parallel_group_path,
13056                });
13057            }
13058
13059            if !signal_wait_events.is_empty() {
13060                let opened: Vec<_> = signal_wait_events
13061                    .iter()
13062                    .copied()
13063                    .filter(|event| event.event_type == "SignalWaitOpened")
13064                    .collect();
13065                if opened.len() != 1 {
13066                    return Err(invalid_recorded_history(
13067                        "signal_wait_open_missing_or_duplicate",
13068                        sequence,
13069                        "one SignalWaitOpened event",
13070                        &format!("{} SignalWaitOpened events", opened.len()),
13071                        "signal replay requires exactly one canonical wait-open event",
13072                    ));
13073                }
13074
13075                let applied: Vec<_> = signal_wait_events
13076                    .iter()
13077                    .copied()
13078                    .filter(|event| event.event_type == "SignalApplied")
13079                    .collect();
13080                if applied.len() > 1 {
13081                    return Err(invalid_recorded_history(
13082                        "duplicate_signal_wait_apply",
13083                        sequence,
13084                        "at most one SignalApplied event",
13085                        "multiple SignalApplied events",
13086                        "signal history applies one durable wait more than once",
13087                    ));
13088                }
13089
13090                let signal_names = signal_wait_events
13091                    .iter()
13092                    .map(|event| required_signal_wait_name(event, sequence))
13093                    .collect::<Result<Vec<_>>>()?;
13094                let signal_name = signal_names
13095                    .first()
13096                    .expect("signal wait events are not empty")
13097                    .clone();
13098                if signal_names.iter().any(|candidate| candidate != &signal_name) {
13099                    return Err(invalid_recorded_history(
13100                        "signal_wait_identity_mismatch",
13101                        sequence,
13102                        &signal_name,
13103                        "conflicting signal names",
13104                        "signal wait lifecycle events at one workflow sequence disagree on identity",
13105                    ));
13106                }
13107                let value = applied
13108                    .first()
13109                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
13110                    .transpose()?;
13111                return Ok(RecordedCommand::SignalWait {
13112                    sequence,
13113                    signal_name,
13114                    value,
13115                    parallel_group_path: recorded_parallel_group_path(
13116                        &signal_wait_events,
13117                        sequence,
13118                    )?,
13119                });
13120            }
13121
13122            if !condition_wait_events.is_empty() {
13123                return recorded_condition_wait(
13124                    sequence,
13125                    &condition_wait_events,
13126                    events,
13127                );
13128            }
13129
13130            if !search_attribute_events.is_empty() {
13131                if search_attribute_events.len() != 1 {
13132                    return Err(invalid_recorded_history(
13133                        "duplicate_search_attribute_update",
13134                        sequence,
13135                        "one SearchAttributesUpserted event",
13136                        &format!(
13137                            "{} SearchAttributesUpserted events",
13138                            search_attribute_events.len()
13139                        ),
13140                        "search-attribute history records one workflow command more than once",
13141                    ));
13142                }
13143                let payload = &search_attribute_events[0].payload;
13144                let attributes = payload
13145                    .get("attributes")
13146                    .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13147                    .cloned()
13148                    .ok_or_else(|| {
13149                        invalid_recorded_history(
13150                            "search_attribute_update_missing",
13151                            sequence,
13152                            "non-empty attributes object",
13153                            "missing or invalid attributes",
13154                            "search-attribute history is missing its recorded mutation",
13155                        )
13156                    })?;
13157                let attribute_types =
13158                    recorded_search_attribute_types(payload, &attributes, sequence)?;
13159                return Ok(RecordedCommand::SearchAttributes {
13160                    sequence,
13161                    attributes,
13162                    attribute_types,
13163                });
13164            }
13165
13166            if !side_effect_events.is_empty() {
13167                if side_effect_events.len() != 1 {
13168                    return Err(invalid_recorded_history(
13169                        "duplicate_side_effect_record",
13170                        sequence,
13171                        "one SideEffectRecorded event",
13172                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
13173                        "side-effect history records one workflow command more than once",
13174                    ));
13175                }
13176                let event = side_effect_events[0];
13177                let result = event.payload.get("result").ok_or_else(|| {
13178                    invalid_recorded_history(
13179                        "side_effect_result_missing",
13180                        sequence,
13181                        "recorded result payload",
13182                        "missing result",
13183                        "side-effect history is missing its recorded value",
13184                    )
13185                })?;
13186                let has_published_envelope = result.as_str().is_some()
13187                    || result.as_object().is_some_and(|envelope| {
13188                        envelope.get("codec").and_then(Value::as_str).is_some()
13189                            && envelope.get("blob").and_then(Value::as_str).is_some()
13190                    });
13191                if !has_published_envelope {
13192                    return Err(invalid_recorded_history(
13193                        "side_effect_payload_malformed",
13194                        sequence,
13195                        "payload blob or {codec, blob} envelope",
13196                        &result.to_string(),
13197                        "side-effect history result does not use a published payload envelope",
13198                    ));
13199                }
13200                let codec = event
13201                    .payload
13202                    .get("payload_codec")
13203                    .and_then(Value::as_str)
13204                    .unwrap_or(fallback_codec);
13205                let value = decode_wire_avro_value(result, codec).map_err(|error| {
13206                    if error.to_string().contains("unsupported_payload_codec") {
13207                        return error;
13208                    }
13209
13210                    invalid_recorded_history(
13211                        "side_effect_payload_incompatible",
13212                        sequence,
13213                        &format!("valid {codec} payload envelope"),
13214                        &error.to_string(),
13215                        "side-effect history payload cannot be decoded with its recorded codec",
13216                    )
13217                })?;
13218                return Ok(RecordedCommand::SideEffect { sequence, value });
13219            }
13220
13221            if !version_marker_events.is_empty() {
13222                if version_marker_events.len() != 1 {
13223                    return Err(invalid_recorded_history(
13224                        "duplicate_version_marker_record",
13225                        sequence,
13226                        "one VersionMarkerRecorded event",
13227                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13228                        "version-marker history records one workflow command more than once",
13229                    ));
13230                }
13231                let payload = &version_marker_events[0].payload;
13232                let change_id = payload
13233                    .get("change_id")
13234                    .and_then(Value::as_str)
13235                    .filter(|value| !value.is_empty())
13236                    .map(str::to_string)
13237                    .ok_or_else(|| {
13238                        invalid_recorded_history(
13239                            "version_marker_field_missing",
13240                            sequence,
13241                            "non-empty change_id",
13242                            "missing or invalid change_id",
13243                            "version-marker history is missing its stable change ID",
13244                        )
13245                    })?;
13246                let version = required_version_i32(payload, "version", sequence)?;
13247                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13248                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13249                if min_supported > max_supported || version < min_supported || version > max_supported {
13250                    return Err(invalid_recorded_history(
13251                        "version_marker_history_range_invalid",
13252                        sequence,
13253                        "min_supported <= version <= max_supported",
13254                        &format!("{min_supported} <= {version} <= {max_supported}"),
13255                        "recorded version marker contains an internally incompatible range",
13256                    ));
13257                }
13258                return Ok(RecordedCommand::VersionMarker {
13259                    sequence,
13260                    change_id,
13261                    version,
13262                });
13263            }
13264
13265            if !memo_events.is_empty() {
13266                if memo_events.len() != 1 {
13267                    return Err(invalid_recorded_history(
13268                        "duplicate_memo_upsert_record",
13269                        sequence,
13270                        "one MemoUpserted event",
13271                        &format!("{} MemoUpserted events", memo_events.len()),
13272                        "memo history records one workflow update more than once",
13273                    ));
13274                }
13275                let payload = &memo_events[0].payload;
13276                let entries = payload.get("entries").cloned().ok_or_else(|| {
13277                    invalid_recorded_history(
13278                        "memo_entries_missing",
13279                        sequence,
13280                        "memo entries object",
13281                        "missing entries",
13282                        "MemoUpserted history is missing replay identity entries",
13283                    )
13284                })?;
13285                let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13286                    invalid_recorded_history(
13287                        "memo_entries_invalid",
13288                        sequence,
13289                        "valid canonical memo entries",
13290                        &error.to_string(),
13291                        "MemoUpserted history contains invalid replay identity entries",
13292                    )
13293                })?;
13294                let merged = payload.get("merged").cloned().ok_or_else(|| {
13295                    invalid_recorded_history(
13296                        "memo_merged_projection_missing",
13297                        sequence,
13298                        "merged memo projection",
13299                        "missing merged",
13300                        "MemoUpserted history is missing its merged projection",
13301                    )
13302                })?;
13303                decode_memo_history_map(&merged, false).map_err(|error| {
13304                    invalid_recorded_history(
13305                        "memo_merged_projection_invalid",
13306                        sequence,
13307                        "valid merged memo projection",
13308                        &error.to_string(),
13309                        "MemoUpserted history contains an invalid merged projection",
13310                    )
13311                })?;
13312
13313                return Ok(RecordedCommand::Memo { sequence, entries });
13314            }
13315            let scheduled: Vec<_> = timer_events
13316                .iter()
13317                .copied()
13318                .filter(|event| event.event_type == "TimerScheduled")
13319                .collect();
13320            let fired: Vec<_> = timer_events
13321                .iter()
13322                .copied()
13323                .filter(|event| event.event_type == "TimerFired")
13324                .collect();
13325            if scheduled.len() != 1 {
13326                return Err(invalid_recorded_history(
13327                    "timer_schedule_missing_or_duplicate",
13328                    sequence,
13329                    "one TimerScheduled event",
13330                    &format!("{} TimerScheduled events", scheduled.len()),
13331                    "timer replay requires exactly one recorded schedule event",
13332                ));
13333            }
13334            if fired.len() > 1 {
13335                return Err(invalid_recorded_history(
13336                    "duplicate_timer_fire",
13337                    sequence,
13338                    "at most one TimerFired event",
13339                    "multiple TimerFired events",
13340                    "timer history contains more than one fire event for a workflow sequence",
13341                ));
13342            }
13343
13344            let scheduled = scheduled[0];
13345            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13346            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13347            if let Some(fired) = fired.first() {
13348                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13349                if fired_timer_id != timer_id {
13350                    return Err(invalid_recorded_history(
13351                        "timer_identity_mismatch",
13352                        sequence,
13353                        &timer_id,
13354                        &fired_timer_id,
13355                        "TimerFired does not correspond to the recorded TimerScheduled event",
13356                    ));
13357                }
13358                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13359                if fired_delay != delay_seconds {
13360                    return Err(invalid_recorded_history(
13361                        "timer_history_delay_mismatch",
13362                        sequence,
13363                        &delay_seconds.to_string(),
13364                        &fired_delay.to_string(),
13365                        "TimerScheduled and TimerFired record different delays",
13366                    ));
13367                }
13368            }
13369
13370            Ok(RecordedCommand::Timer {
13371                sequence,
13372                delay_seconds,
13373                fired: !fired.is_empty(),
13374                parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13375            })
13376        })
13377        .collect::<Result<_>>()?;
13378
13379    let mut marker_sequences = HashMap::new();
13380    for command in &commands {
13381        if let RecordedCommand::VersionMarker {
13382            sequence,
13383            change_id,
13384            ..
13385        } = command
13386        {
13387            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13388                return Err(invalid_recorded_history(
13389                    "duplicate_version_marker",
13390                    *sequence,
13391                    &format!("one marker for change ID {change_id:?}"),
13392                    &format!("markers at sequences {first_sequence} and {sequence}"),
13393                    "workflow history contains duplicate markers for one stable change ID",
13394                ));
13395            }
13396        }
13397    }
13398
13399    Ok(commands)
13400}
13401
13402fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13403    payload
13404        .get(field)
13405        .and_then(Value::as_i64)
13406        .and_then(|value| i32::try_from(value).ok())
13407        .ok_or_else(|| {
13408            invalid_recorded_history(
13409                "version_marker_field_missing",
13410                sequence,
13411                &format!("integer {field}"),
13412                "missing or out-of-range integer",
13413                "version-marker history is missing a required integer field",
13414            )
13415        })
13416}
13417
13418fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13419    event
13420        .payload
13421        .get("sequence")
13422        .or_else(|| event.payload.get("workflow_sequence"))
13423        .or_else(|| event.raw.get("sequence"))
13424        .or_else(|| event.raw.get("workflow_sequence"))
13425        .and_then(value_as_u64)
13426}
13427
13428fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13429    matches!(
13430        event
13431            .payload
13432            .get("timer_kind")
13433            .or_else(|| event.raw.get("timer_kind"))
13434            .and_then(Value::as_str),
13435        Some("condition_timeout" | "signal_timeout")
13436    )
13437}
13438
13439fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13440    matches!(
13441        event.event_type.as_str(),
13442        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13443    )
13444}
13445
13446fn recorded_condition_wait(
13447    sequence: u64,
13448    condition_events: &[&HistoryEvent],
13449    all_events: &[HistoryEvent],
13450) -> Result<RecordedCommand> {
13451    let opened = condition_events
13452        .iter()
13453        .copied()
13454        .filter(|event| event.event_type == "ConditionWaitOpened")
13455        .collect::<Vec<_>>();
13456    if opened.len() != 1 {
13457        return Err(invalid_recorded_history(
13458            "condition_wait_open_missing_or_duplicate",
13459            sequence,
13460            "one ConditionWaitOpened event",
13461            &format!("{} ConditionWaitOpened events", opened.len()),
13462            "condition replay requires exactly one canonical wait-open event",
13463        ));
13464    }
13465    let terminal = condition_events
13466        .iter()
13467        .copied()
13468        .filter(|event| {
13469            matches!(
13470                event.event_type.as_str(),
13471                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13472            )
13473        })
13474        .collect::<Vec<_>>();
13475    if terminal.len() > 1 {
13476        return Err(invalid_recorded_history(
13477            "duplicate_condition_wait_terminal_event",
13478            sequence,
13479            "at most one condition terminal event",
13480            "multiple condition terminal events",
13481            "condition history settles one durable wait more than once",
13482        ));
13483    }
13484
13485    let opened = opened[0];
13486    let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13487    let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13488    for event in condition_events
13489        .iter()
13490        .copied()
13491        .filter(|event| !std::ptr::eq(*event, opened))
13492    {
13493        let event_wait_id = required_condition_wait_id(event, sequence)?;
13494        if event_wait_id != condition_wait_id {
13495            return Err(invalid_recorded_history(
13496                "condition_wait_id_mismatch",
13497                sequence,
13498                &condition_wait_id,
13499                &event_wait_id,
13500                "condition lifecycle events at one sequence disagree on wait identity",
13501            ));
13502        }
13503        let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13504        if event_occurrence_id != occurrence_id {
13505            return Err(invalid_recorded_history(
13506                "condition_wait_occurrence_history_mismatch",
13507                sequence,
13508                &occurrence_id,
13509                &event_occurrence_id,
13510                "condition lifecycle events at one sequence disagree on authored occurrence identity",
13511            ));
13512        }
13513    }
13514
13515    let condition_key = optional_non_empty_history_string(opened, "condition_key");
13516    let predicate_identity = opened
13517        .payload
13518        .get("condition_definition_fingerprint")
13519        .and_then(Value::as_str)
13520        .filter(|value| !value.is_empty())
13521        .map(str::to_string)
13522        .ok_or_else(|| {
13523            invalid_recorded_history(
13524                "condition_wait_predicate_fingerprint_missing",
13525                sequence,
13526                "non-empty condition_definition_fingerprint",
13527                &opened.event_type,
13528                "canonical condition history is missing its predicate identity",
13529            )
13530        })?;
13531    let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13532    for event in condition_events
13533        .iter()
13534        .copied()
13535        .filter(|event| !std::ptr::eq(*event, opened))
13536    {
13537        for (field, opened_value) in [
13538            ("condition_key", condition_key.as_deref()),
13539            (
13540                "condition_definition_fingerprint",
13541                Some(predicate_identity.as_str()),
13542            ),
13543        ] {
13544            if let Some(value) = optional_non_empty_history_string(event, field) {
13545                if opened_value.is_some_and(|opened_value| opened_value != value) {
13546                    return Err(invalid_recorded_history(
13547                        "condition_wait_definition_history_mismatch",
13548                        sequence,
13549                        opened_value.unwrap_or_default(),
13550                        &value,
13551                        "condition lifecycle events disagree on the recorded definition",
13552                    ));
13553                }
13554            }
13555        }
13556        if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13557            if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13558                return Err(invalid_recorded_history(
13559                    "condition_wait_definition_history_mismatch",
13560                    sequence,
13561                    &format!("{}s", timeout_seconds.unwrap_or_default()),
13562                    &format!("{event_timeout}s"),
13563                    "condition lifecycle events disagree on the recorded timeout",
13564                ));
13565            }
13566        }
13567    }
13568
13569    let timeout_timer_events = all_events
13570        .iter()
13571        .filter(|event| {
13572            matches!(
13573                event.event_type.as_str(),
13574                "TimerScheduled" | "TimerCancelled" | "TimerFired"
13575            ) && event.payload.get("timer_kind").and_then(Value::as_str)
13576                == Some("condition_timeout")
13577                && event
13578                    .payload
13579                    .get("condition_wait_id")
13580                    .and_then(Value::as_str)
13581                    == Some(condition_wait_id.as_str())
13582        })
13583        .collect::<Vec<_>>();
13584    let scheduled = timeout_timer_events
13585        .iter()
13586        .copied()
13587        .filter(|event| event.event_type == "TimerScheduled")
13588        .collect::<Vec<_>>();
13589    let fired = timeout_timer_events
13590        .iter()
13591        .copied()
13592        .filter(|event| event.event_type == "TimerFired")
13593        .collect::<Vec<_>>();
13594    if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13595        return Err(invalid_recorded_history(
13596            "condition_wait_timeout_history_invalid",
13597            sequence,
13598            "one timeout schedule and at most one fire",
13599            &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13600            "condition timeout history has a missing or duplicate lifecycle event",
13601        ));
13602    }
13603    if let Some(scheduled) = scheduled.first() {
13604        let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13605        let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13606        if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13607            return Err(invalid_recorded_history(
13608                "condition_wait_timeout_delay_mismatch",
13609                sequence,
13610                &format!("{}s", timeout_seconds.unwrap_or_default()),
13611                &format!("{delay_seconds}s"),
13612                "condition timeout timer differs from the wait definition",
13613            ));
13614        }
13615        if let Some(fired) = fired.first() {
13616            let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13617            let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13618            if fired_timer_id != timer_id || fired_delay != delay_seconds {
13619                return Err(invalid_recorded_history(
13620                    "condition_wait_timeout_identity_mismatch",
13621                    sequence,
13622                    &format!("{timer_id}:{delay_seconds}s"),
13623                    &format!("{fired_timer_id}:{fired_delay}s"),
13624                    "condition timeout fire does not match its durable schedule",
13625                ));
13626            }
13627        }
13628    }
13629
13630    let result = terminal.first().map(|event| {
13631        if event.event_type == "ConditionWaitTimedOut" {
13632            ConditionWaitResult::TimedOut
13633        } else {
13634            ConditionWaitResult::Satisfied
13635        }
13636    });
13637    let result = if !fired.is_empty() {
13638        if result == Some(ConditionWaitResult::Satisfied) {
13639            return Err(invalid_recorded_history(
13640                "condition_wait_terminal_conflict",
13641                sequence,
13642                "one satisfied or timed-out outcome",
13643                "satisfied event and fired timeout",
13644                "condition history records conflicting terminal outcomes",
13645            ));
13646        }
13647        Some(ConditionWaitResult::TimedOut)
13648    } else {
13649        result
13650    };
13651
13652    Ok(RecordedCommand::ConditionWait {
13653        sequence,
13654        occurrence_id,
13655        condition_key,
13656        predicate_identity,
13657        timeout_seconds,
13658        result,
13659        parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13660    })
13661}
13662
13663fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13664    event
13665        .payload
13666        .get("condition_wait_occurrence_id")
13667        .and_then(Value::as_str)
13668        .filter(|value| !value.is_empty())
13669        .map(str::to_string)
13670        .ok_or_else(|| {
13671            invalid_recorded_history(
13672                "condition_wait_occurrence_id_missing",
13673                sequence,
13674                "non-empty condition_wait_occurrence_id",
13675                &event.event_type,
13676                "condition history is missing authored occurrence identity",
13677            )
13678        })
13679}
13680
13681fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13682    event
13683        .payload
13684        .get("condition_wait_id")
13685        .and_then(Value::as_str)
13686        .filter(|value| !value.is_empty())
13687        .map(str::to_string)
13688        .ok_or_else(|| {
13689            invalid_recorded_history(
13690                "condition_wait_id_missing",
13691                sequence,
13692                "non-empty condition_wait_id",
13693                &event.event_type,
13694                "canonical condition history is missing its durable wait identity",
13695            )
13696        })
13697}
13698
13699fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13700    event
13701        .payload
13702        .get(field)
13703        .and_then(Value::as_str)
13704        .filter(|value| !value.is_empty())
13705        .map(str::to_string)
13706}
13707
13708fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13709    match event.payload.get(field) {
13710        None | Some(Value::Null) => Ok(None),
13711        Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13712            invalid_recorded_history(
13713                "condition_wait_definition_invalid",
13714                sequence,
13715                &format!("non-negative integer {field}"),
13716                &value.to_string(),
13717                "condition history contains an invalid numeric definition field",
13718            )
13719        }),
13720    }
13721}
13722
13723fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13724    event
13725        .payload
13726        .get("signal_name")
13727        .or_else(|| event.raw.get("signal_name"))
13728        .and_then(Value::as_str)
13729        .filter(|value| !value.is_empty())
13730        .map(str::to_string)
13731        .ok_or_else(|| {
13732            invalid_recorded_history(
13733                "signal_wait_name_missing",
13734                sequence,
13735                "non-empty signal_name",
13736                &event.event_type,
13737                "canonical signal-wait history is missing its signal identity",
13738            )
13739        })
13740}
13741
13742fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13743    matches!(
13744        event.event_type.as_str(),
13745        "SignalWaitOpened" | "SignalApplied"
13746    )
13747}
13748
13749fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
13750    event
13751        .payload
13752        .get(field)
13753        .and_then(Value::as_str)
13754        .filter(|value| !value.is_empty())
13755        .map(str::to_string)
13756        .ok_or_else(|| {
13757            invalid_recorded_history(
13758                "timer_history_field_missing",
13759                sequence,
13760                field,
13761                &event.event_type,
13762                "timer history is missing a required identity field",
13763            )
13764        })
13765}
13766
13767fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
13768    event
13769        .payload
13770        .get(field)
13771        .and_then(value_as_u64)
13772        .ok_or_else(|| {
13773            invalid_recorded_history(
13774                "timer_history_field_missing",
13775                sequence,
13776                field,
13777                &event.event_type,
13778                "timer history is missing a required numeric field",
13779            )
13780        })
13781}
13782
13783fn recorded_search_attribute_types(
13784    payload: &Value,
13785    attributes: &Value,
13786    sequence: u64,
13787) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
13788    let Some(raw_types) = payload.get("attribute_types") else {
13789        // This is the explicit compatibility rule for histories recorded
13790        // before typed identity was persisted. Values still constrain replay;
13791        // the unknown type snapshot does not assert a typed match.
13792        return Ok(RecordedSnapshotValue::Unknown);
13793    };
13794    let Some(raw_types) = raw_types.as_object() else {
13795        return Err(invalid_recorded_history(
13796            "search_attribute_types_malformed",
13797            sequence,
13798            "canonical attribute type map",
13799            &raw_types.to_string(),
13800            "search-attribute history contains malformed type identity",
13801        ));
13802    };
13803    let attribute_keys = attributes
13804        .as_object()
13805        .expect("recorded search attributes were validated as an object");
13806    let mut types = BTreeMap::new();
13807    for (key, value) in raw_types {
13808        let Some(attribute_type) = value.as_str() else {
13809            return Err(invalid_recorded_history(
13810                "search_attribute_types_malformed",
13811                sequence,
13812                "canonical string type name",
13813                &value.to_string(),
13814                "search-attribute history contains a non-string type identity",
13815            ));
13816        };
13817        if !attribute_keys.contains_key(key)
13818            || !matches!(
13819                attribute_type,
13820                "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
13821            )
13822        {
13823            return Err(invalid_recorded_history(
13824                "search_attribute_types_malformed",
13825                sequence,
13826                "canonical types for keys present in attributes",
13827                &format!("{key}:{attribute_type}"),
13828                "search-attribute history contains unsupported or orphaned type identity",
13829            ));
13830        }
13831        types.insert(key.clone(), attribute_type.to_string());
13832    }
13833    Ok(RecordedSnapshotValue::Known(types))
13834}
13835
13836fn invalid_recorded_history(
13837    reason: &str,
13838    sequence: u64,
13839    expected: &str,
13840    actual: &str,
13841    message: &str,
13842) -> Error {
13843    Error::NonDeterministicReplay(ReplayFailure::new(
13844        reason,
13845        Some(sequence),
13846        Some(expected.to_string()),
13847        Some(actual.to_string()),
13848        message,
13849    ))
13850}
13851
13852type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
13853
13854fn activity_outcome(
13855    event: &HistoryEvent,
13856    fallback_codec: &str,
13857    recorded_activity_type: Option<String>,
13858) -> Result<ActivityOutcome> {
13859    if event.event_type == "ActivityCompleted" {
13860        let codec = event
13861            .payload
13862            .get("payload_codec")
13863            .and_then(Value::as_str)
13864            .unwrap_or(fallback_codec);
13865        return Ok(Ok(decode_wire_avro_value(
13866            event.payload.get("result").unwrap_or(&Value::Null),
13867            codec,
13868        )?));
13869    }
13870
13871    let payload = &event.payload;
13872    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
13873        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
13874        "ActivityCancelled" => (
13875            ActivityFailureKind::Cancelled,
13876            "cancelled",
13877            "activity was cancelled",
13878        ),
13879        "ActivityTimedOut" => (
13880            ActivityFailureKind::TimedOut,
13881            "timeout",
13882            "activity timed out",
13883        ),
13884        _ => unreachable!("activity_outcome is called only for terminal activity events"),
13885    };
13886    let exception = payload
13887        .get("exception")
13888        .filter(|value| !value.is_null())
13889        .cloned();
13890    let failure_category = payload_string(payload, "failure_category");
13891    let timeout_kind = payload_string(payload, "timeout_kind");
13892    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
13893        ActivityFailureKind::Failed => failure_category
13894            .clone()
13895            .unwrap_or_else(|| fallback_reason.to_string()),
13896        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
13897        ActivityFailureKind::TimedOut => timeout_kind
13898            .clone()
13899            .unwrap_or_else(|| fallback_reason.to_string()),
13900    });
13901    let message = payload_string(payload, "message")
13902        .or_else(|| {
13903            exception
13904                .as_ref()
13905                .and_then(|value| payload_string(value, "message"))
13906        })
13907        .unwrap_or_else(|| fallback_message.to_string());
13908
13909    Ok(Err(ActivityFailure {
13910        kind,
13911        reason,
13912        message,
13913        activity_execution_id: payload_string(payload, "activity_execution_id"),
13914        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
13915        activity_type: payload_string(payload, "activity_type")
13916            .or_else(|| payload_string(payload, "activity_name"))
13917            .or(recorded_activity_type),
13918        activity_class: payload_string(payload, "activity_class"),
13919        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
13920        failure_id: payload_string(payload, "failure_id"),
13921        failure_category,
13922        timeout_kind,
13923        non_retryable: payload
13924            .get("non_retryable")
13925            .and_then(Value::as_bool)
13926            .unwrap_or(false),
13927        exception_type: payload_string(payload, "exception_type").or_else(|| {
13928            exception
13929                .as_ref()
13930                .and_then(|value| payload_string(value, "type"))
13931        }),
13932        exception_class: payload_string(payload, "exception_class").or_else(|| {
13933            exception
13934                .as_ref()
13935                .and_then(|value| payload_string(value, "class"))
13936        }),
13937        code: payload
13938            .get("code")
13939            .filter(|value| !value.is_null())
13940            .cloned(),
13941        exception,
13942    }))
13943}
13944
13945type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
13946
13947fn child_workflow_outcomes(
13948    events: &[HistoryEvent],
13949    fallback_codec: &str,
13950    parent: WorkflowIdentity,
13951) -> Result<Vec<ChildWorkflowOutcome>> {
13952    let mut outcomes = Vec::new();
13953
13954    for event in events {
13955        let kind = match event.event_type.as_str() {
13956            "ChildRunCompleted" => None,
13957            "ChildRunFailed" => Some((
13958                ChildWorkflowFailureKind::Failed,
13959                "child_workflow",
13960                "child workflow failed",
13961            )),
13962            "ChildRunCancelled" => Some((
13963                ChildWorkflowFailureKind::Cancelled,
13964                "cancelled",
13965                "child workflow was cancelled",
13966            )),
13967            "ChildRunTerminated" => Some((
13968                ChildWorkflowFailureKind::Terminated,
13969                "terminated",
13970                "child workflow was terminated",
13971            )),
13972            _ => continue,
13973        };
13974        let payload = &event.payload;
13975        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
13976        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
13977        let child_workflow_type = payload_string(payload, "child_workflow_type");
13978
13979        if let Some((kind, reason, fallback_message)) = kind {
13980            let exception = payload
13981                .get("exception")
13982                .filter(|value| !value.is_null())
13983                .cloned();
13984            let message = payload_string(payload, "message")
13985                .or_else(|| {
13986                    exception
13987                        .as_ref()
13988                        .and_then(|value| payload_string(value, "message"))
13989                })
13990                .unwrap_or_else(|| fallback_message.to_string());
13991            let exception_type = payload_string(payload, "exception_type").or_else(|| {
13992                exception
13993                    .as_ref()
13994                    .and_then(|value| payload_string(value, "type"))
13995            });
13996            let exception_class = payload_string(payload, "exception_class").or_else(|| {
13997                exception
13998                    .as_ref()
13999                    .and_then(|value| payload_string(value, "class"))
14000            });
14001            outcomes.push(Err(ChildWorkflowFailure {
14002                kind,
14003                reason: reason.to_string(),
14004                message,
14005                parent_workflow_id: parent.workflow_id.clone(),
14006                parent_workflow_run_id: parent.run_id.clone(),
14007                child_workflow_id,
14008                child_workflow_run_id,
14009                child_workflow_type,
14010                failure_id: payload_string(payload, "failure_id"),
14011                failure_category: payload_string(payload, "failure_category"),
14012                exception_type,
14013                exception_class,
14014                non_retryable: payload
14015                    .get("non_retryable")
14016                    .and_then(Value::as_bool)
14017                    .unwrap_or(false),
14018                code: payload
14019                    .get("code")
14020                    .filter(|value| !value.is_null())
14021                    .cloned(),
14022                exception,
14023            }));
14024            continue;
14025        }
14026
14027        let codec = payload
14028            .get("payload_codec")
14029            .and_then(Value::as_str)
14030            .unwrap_or(fallback_codec);
14031        let result = payload
14032            .get("result")
14033            .or_else(|| payload.get("output"))
14034            .unwrap_or(&Value::Null);
14035        outcomes.push(Ok(ChildWorkflowAvroResult {
14036            parent: parent.clone(),
14037            child: WorkflowIdentity {
14038                workflow_id: child_workflow_id,
14039                run_id: child_workflow_run_id,
14040            },
14041            child_workflow_type,
14042            result: decode_wire_avro_value(result, codec)?,
14043        }));
14044    }
14045
14046    Ok(outcomes)
14047}
14048
14049fn payload_string(payload: &Value, key: &str) -> Option<String> {
14050    payload
14051        .get(key)
14052        .and_then(Value::as_str)
14053        .filter(|value| !value.is_empty())
14054        .map(str::to_string)
14055}
14056
14057fn workflow_failure_command(error: &Error) -> Value {
14058    let (exception_type, exception_class, properties) = match error {
14059        Error::ActivityFailed(failure) => (
14060            match failure.kind {
14061                ActivityFailureKind::Failed => "ActivityFailed",
14062                ActivityFailureKind::Cancelled => "ActivityCancelled",
14063                ActivityFailureKind::TimedOut => "ActivityTimedOut",
14064            },
14065            "durable_workflow::ActivityFailure",
14066            json!({
14067                "reason": failure.reason,
14068                "activity_execution_id": failure.activity_execution_id,
14069                "activity_attempt_id": failure.activity_attempt_id,
14070                "activity_type": failure.activity_type,
14071                "activity_class": failure.activity_class,
14072                "attempt_number": failure.attempt_number,
14073                "failure_id": failure.failure_id,
14074                "failure_category": failure.failure_category,
14075                "timeout_kind": failure.timeout_kind,
14076                "activity_non_retryable": failure.non_retryable,
14077                "activity_exception_type": failure.exception_type,
14078                "activity_exception_class": failure.exception_class,
14079                "activity_code": failure.code,
14080                "activity_exception": failure.exception,
14081            }),
14082        ),
14083        Error::ChildWorkflowFailed(failure) => (
14084            match failure.kind {
14085                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14086                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14087                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14088            },
14089            "durable_workflow::ChildWorkflowFailure",
14090            json!({
14091                "reason": failure.reason,
14092                "parent_workflow_id": failure.parent_workflow_id,
14093                "parent_workflow_run_id": failure.parent_workflow_run_id,
14094                "child_workflow_id": failure.child_workflow_id,
14095                "child_workflow_run_id": failure.child_workflow_run_id,
14096                "child_workflow_type": failure.child_workflow_type,
14097                "failure_id": failure.failure_id,
14098                "failure_category": failure.failure_category,
14099                "child_exception_type": failure.exception_type,
14100                "child_exception_class": failure.exception_class,
14101                "child_non_retryable": failure.non_retryable,
14102                "child_code": failure.code,
14103                "child_exception": failure.exception,
14104            }),
14105        ),
14106        Error::ParallelFailed(failure) => (
14107            "ParallelFailed",
14108            "durable_workflow::ParallelFailure",
14109            json!({
14110                "parallel_group_id": failure.group_id,
14111                "parallel_member_path": failure.member_path,
14112                "parallel_group_path": failure.group_path,
14113                "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14114                "cause_type": workflow_error_type(&failure.cause),
14115                "cause_message": failure.cause.to_string(),
14116            }),
14117        ),
14118        Error::SagaCompensationFailed(failure) => (
14119            "SagaCompensationFailed",
14120            "durable_workflow::SagaCompensationFailure",
14121            json!({
14122                "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14123                "initiating_failure_message": failure.initiating_failure.to_string(),
14124                "compensation_activity_type": failure.compensation_activity_type,
14125                "compensation_registration_order": failure.compensation_registration_order,
14126                "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14127                "compensation_failure_message": failure.compensation_failure.to_string(),
14128            }),
14129        ),
14130        Error::WorkflowCancellationRequested(_) => (
14131            "WorkflowCancellationRequested",
14132            "durable_workflow::WorkflowCancellationRequested",
14133            json!({"reason": "cancelled"}),
14134        ),
14135        Error::NonDeterministicReplay(_) => (
14136            "NonDeterministicReplay",
14137            "durable_workflow::Error",
14138            Value::Null,
14139        ),
14140        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14141    };
14142    let non_retryable = match error {
14143        Error::ActivityFailed(failure) => failure.non_retryable,
14144        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14145        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14146        Error::SagaCompensationFailed(failure) => {
14147            workflow_error_non_retryable(&failure.compensation_failure)
14148        }
14149        Error::WorkflowCancellationRequested(_) => true,
14150        Error::NonDeterministicReplay(_) => true,
14151        _ => false,
14152    };
14153
14154    json!({
14155        "type": "fail_workflow",
14156        "message": error.to_string(),
14157        "exception_type": exception_type,
14158        "exception_class": exception_class,
14159        "non_retryable": non_retryable,
14160        "exception": {
14161            "type": exception_type,
14162            "class": exception_class,
14163            "message": error.to_string(),
14164            "properties": properties,
14165        }
14166    })
14167}
14168
14169fn workflow_error_type(error: &Error) -> &'static str {
14170    match error {
14171        Error::ActivityFailed(failure) => match failure.kind {
14172            ActivityFailureKind::Failed => "ActivityFailed",
14173            ActivityFailureKind::Cancelled => "ActivityCancelled",
14174            ActivityFailureKind::TimedOut => "ActivityTimedOut",
14175        },
14176        Error::ChildWorkflowFailed(failure) => match failure.kind {
14177            ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14178            ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14179            ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14180        },
14181        Error::ParallelFailed(_) => "ParallelFailed",
14182        Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14183        Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14184        Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14185        _ => "RustWorkflowError",
14186    }
14187}
14188
14189fn workflow_error_non_retryable(error: &Error) -> bool {
14190    match error {
14191        Error::ActivityFailed(failure) => failure.non_retryable,
14192        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14193        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14194        Error::SagaCompensationFailed(failure) => {
14195            workflow_error_non_retryable(&failure.compensation_failure)
14196        }
14197        Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14198        _ => false,
14199    }
14200}
14201
14202fn workflow_task_integrity_error(error: &Error) -> bool {
14203    matches!(
14204        error,
14205        Error::NonDeterministicReplay(_)
14206            | Error::Protocol(_)
14207            | Error::MissingWorkflowCommandIdentity
14208            | Error::WorkflowStatePoisoned
14209    )
14210}
14211
14212fn decode_signal_event_arguments(
14213    event: &HistoryEvent,
14214    fallback_codec: &str,
14215) -> Result<Vec<AvroValue>> {
14216    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14217    validate_payload_codec(codec)?;
14218    let raw = signal_history_payload(&event.payload);
14219    let decoded = match raw.filter(|value| !value.is_null()) {
14220        Some(value) => decode_wire_avro_value(value, codec)?,
14221        None => AvroValue::Array(Vec::new()),
14222    };
14223    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14224        unreachable!("normalize_avro_arguments always returns an array");
14225    };
14226    Ok(arguments)
14227}
14228
14229fn decode_update_event_arguments(
14230    event: &HistoryEvent,
14231    fallback_codec: &str,
14232) -> Result<Vec<AvroValue>> {
14233    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14234    validate_payload_codec(codec)?;
14235    let decoded = match event
14236        .payload
14237        .get("arguments")
14238        .filter(|value| !value.is_null())
14239    {
14240        Some(value) => decode_wire_avro_value(value, codec)?,
14241        None => AvroValue::Array(Vec::new()),
14242    };
14243    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14244        unreachable!("normalize_avro_arguments always returns an array");
14245    };
14246    Ok(arguments)
14247}
14248
14249fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14250    let Some(export_events) = task
14251        .history_export
14252        .as_ref()
14253        .and_then(|export| export.get("history_events"))
14254        .and_then(Value::as_array)
14255    else {
14256        return Ok(());
14257    };
14258
14259    if export_events.len() > task.history_events.len() {
14260        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14261    }
14262
14263    Ok(())
14264}
14265
14266fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14267    let Some(export) = task.history_export.as_ref() else {
14268        return Ok(());
14269    };
14270    let signals = export
14271        .get("signals")
14272        .and_then(Value::as_array)
14273        .cloned()
14274        .unwrap_or_default();
14275    let activities = export
14276        .get("activities")
14277        .and_then(Value::as_array)
14278        .cloned()
14279        .unwrap_or_default();
14280    let export_codec = export
14281        .get("payloads")
14282        .and_then(|payloads| payloads.get("codec"))
14283        .and_then(Value::as_str)
14284        .unwrap_or(&task.payload_codec)
14285        .to_string();
14286    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14287
14288    for event in &mut task.history_events {
14289        if event.event_type == "ActivityCompleted" {
14290            let sequence = event
14291                .payload
14292                .get("sequence")
14293                .or_else(|| event.payload.get("workflow_sequence"))
14294                .and_then(value_as_u64);
14295            let Some(activity) = sequence.and_then(|sequence| {
14296                activities.iter().find(|activity| {
14297                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14298                })
14299            }) else {
14300                continue;
14301            };
14302            let Some(payload) = event.payload.as_object_mut() else {
14303                continue;
14304            };
14305            if missing_payload(payload.get("result")) {
14306                if let Some(result) = activity
14307                    .get("result")
14308                    .filter(|value| !missing_payload(Some(value)))
14309                {
14310                    payload.insert("result".to_string(), result.clone());
14311                }
14312            }
14313            for field in ["payload_codec", "activity_type"] {
14314                if payload
14315                    .get(field)
14316                    .and_then(Value::as_str)
14317                    .unwrap_or_default()
14318                    .is_empty()
14319                {
14320                    if let Some(value) = activity.get(field) {
14321                        payload.insert(field.to_string(), value.clone());
14322                    }
14323                }
14324            }
14325            continue;
14326        }
14327
14328        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14329            continue;
14330        }
14331        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14332        let command_id = event
14333            .payload
14334            .get("workflow_command_id")
14335            .or_else(|| event.raw.get("workflow_command_id"))
14336            .and_then(Value::as_str);
14337        let signal_name = event
14338            .payload
14339            .get("signal_name")
14340            .and_then(Value::as_str)
14341            .unwrap_or_default()
14342            .to_string();
14343        let matched = signals
14344            .iter()
14345            .find(|signal| {
14346                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14347            })
14348            .or_else(|| {
14349                signals.iter().find(|signal| {
14350                    command_id.is_some()
14351                        && signal.get("command_id").and_then(Value::as_str) == command_id
14352                })
14353            })
14354            .or_else(|| {
14355                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14356                let signal = signals
14357                    .iter()
14358                    .filter(|signal| {
14359                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14360                    })
14361                    .nth(*offset);
14362                if signal.is_some() {
14363                    *offset += 1;
14364                }
14365                signal
14366            });
14367        let Some(signal) = matched else {
14368            continue;
14369        };
14370        let signal_codec = signal
14371            .get("payload_codec")
14372            .and_then(Value::as_str)
14373            .unwrap_or(&export_codec);
14374        let Some(payload) = event.payload.as_object_mut() else {
14375            continue;
14376        };
14377        if missing_payload(payload.get("arguments")) {
14378            if let Some(arguments) = signal
14379                .get("arguments")
14380                .filter(|value| !missing_payload(Some(value)))
14381            {
14382                let envelope = match arguments {
14383                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14384                    other => other.clone(),
14385                };
14386                payload.insert("arguments".to_string(), envelope);
14387            }
14388        }
14389        if payload
14390            .get("payload_codec")
14391            .and_then(Value::as_str)
14392            .unwrap_or_default()
14393            .is_empty()
14394        {
14395            payload.insert("payload_codec".to_string(), json!(signal_codec));
14396        }
14397    }
14398
14399    Ok(())
14400}
14401
14402fn missing_payload(value: Option<&Value>) -> bool {
14403    match value {
14404        None | Some(Value::Null) => true,
14405        Some(Value::String(value)) => value.is_empty(),
14406        Some(_) => false,
14407    }
14408}
14409
14410fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14411    let export_signals = task
14412        .history_export
14413        .as_ref()
14414        .and_then(|export| export.get("signals"))
14415        .and_then(Value::as_array)
14416        .cloned()
14417        .unwrap_or_default();
14418    let export_codec = task
14419        .history_export
14420        .as_ref()
14421        .and_then(|export| export.get("payloads"))
14422        .and_then(|payloads| payloads.get("codec"))
14423        .and_then(Value::as_str)
14424        .unwrap_or(&task.payload_codec);
14425    let mut name_offsets: HashMap<String, usize> = HashMap::new();
14426    let mut signals = Vec::new();
14427
14428    for event in &task.history_events {
14429        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14430            continue;
14431        }
14432
14433        let name = event
14434            .payload
14435            .get("signal_name")
14436            .and_then(Value::as_str)
14437            .unwrap_or_default();
14438        if name.is_empty() {
14439            continue;
14440        }
14441        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14442        let command_id = event
14443            .payload
14444            .get("workflow_command_id")
14445            .or_else(|| event.raw.get("workflow_command_id"))
14446            .and_then(Value::as_str);
14447        let matched_export = export_signals
14448            .iter()
14449            .find(|candidate| {
14450                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14451            })
14452            .or_else(|| {
14453                export_signals.iter().find(|candidate| {
14454                    command_id.is_some()
14455                        && candidate.get("command_id").and_then(Value::as_str) == command_id
14456                })
14457            })
14458            .or_else(|| {
14459                let offset = name_offsets.entry(name.to_string()).or_default();
14460                let candidate = export_signals
14461                    .iter()
14462                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14463                    .nth(*offset);
14464                if candidate.is_some() {
14465                    *offset += 1;
14466                }
14467                candidate
14468            });
14469        let codec = event
14470            .payload
14471            .get("payload_codec")
14472            .and_then(Value::as_str)
14473            .or_else(|| {
14474                matched_export
14475                    .and_then(|signal| signal.get("payload_codec"))
14476                    .and_then(Value::as_str)
14477            })
14478            .unwrap_or(export_codec);
14479        let raw_arguments = signal_history_payload(&event.payload)
14480            .filter(|value| !value.is_null())
14481            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14482        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14483        let workflow_sequence = event
14484            .payload
14485            .get("workflow_sequence")
14486            .and_then(value_as_u64)
14487            .or_else(|| {
14488                matched_export
14489                    .and_then(|signal| signal.get("workflow_sequence"))
14490                    .and_then(value_as_u64)
14491            });
14492
14493        signals.push(QuerySignal {
14494            id: signal_id.map(str::to_string).or_else(|| {
14495                matched_export
14496                    .and_then(|signal| signal.get("id"))
14497                    .and_then(Value::as_str)
14498                    .map(str::to_string)
14499            }),
14500            name: name.to_string(),
14501            arguments,
14502            avro_arguments,
14503            workflow_sequence,
14504        });
14505    }
14506
14507    if signals.is_empty() {
14508        for signal in export_signals {
14509            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14510                continue;
14511            }
14512            let Some(name) = signal.get("name").and_then(Value::as_str) else {
14513                continue;
14514            };
14515            let codec = signal
14516                .get("payload_codec")
14517                .and_then(Value::as_str)
14518                .unwrap_or(export_codec);
14519            let (arguments, avro_arguments) =
14520                decode_query_signal_arguments(signal.get("arguments"), codec)?;
14521            signals.push(QuerySignal {
14522                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14523                name: name.to_string(),
14524                arguments,
14525                avro_arguments,
14526                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14527            });
14528        }
14529        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14530    }
14531
14532    Ok(signals)
14533}
14534
14535fn decode_query_signal_arguments(
14536    raw: Option<&Value>,
14537    codec: &str,
14538) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14539    validate_payload_codec(codec)?;
14540    let decoded = match raw.filter(|value| !value.is_null()) {
14541        Some(value) => decode_wire_avro_value(value, codec)?,
14542        None => AvroValue::Array(Vec::new()),
14543    };
14544    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14545        unreachable!("normalize_avro_arguments always returns an array");
14546    };
14547    let arguments = avro_arguments
14548        .iter()
14549        .cloned()
14550        .map(AvroValue::into_json)
14551        .collect::<Result<Vec<_>>>()?;
14552    Ok((arguments, avro_arguments))
14553}
14554
14555fn value_as_u64(value: &Value) -> Option<u64> {
14556    value
14557        .as_u64()
14558        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14559}
14560
14561#[cfg(test)]
14562mod tests {
14563    use super::*;
14564    use std::{
14565        fs,
14566        io::{Read, Write},
14567        net::{SocketAddr, TcpListener, TcpStream},
14568        process::Command as ProcessCommand,
14569        sync::atomic::AtomicUsize,
14570        thread,
14571    };
14572
14573    #[derive(Clone, Copy, Debug)]
14574    enum InvalidTaskPayloadCodec {
14575        Missing,
14576        Null,
14577        NonString,
14578    }
14579
14580    impl InvalidTaskPayloadCodec {
14581        fn label(self) -> &'static str {
14582            match self {
14583                Self::Missing => "missing",
14584                Self::Null => "null",
14585                Self::NonString => "non-string",
14586            }
14587        }
14588
14589        fn apply(self, task: &mut Value) {
14590            let task = task.as_object_mut().expect("task fixture object");
14591            match self {
14592                Self::Missing => {
14593                    task.remove("payload_codec");
14594                }
14595                Self::Null => {
14596                    task.insert("payload_codec".to_string(), Value::Null);
14597                }
14598                Self::NonString => {
14599                    task.insert("payload_codec".to_string(), json!(42));
14600                }
14601            }
14602        }
14603    }
14604
14605    fn fixture_envelope(value: Value) -> Value {
14606        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14607    }
14608
14609    fn fixture_blob(value: Value) -> String {
14610        encode_payload(&value, DEFAULT_CODEC)
14611            .expect("encode Avro test fixture")
14612            .blob
14613    }
14614
14615    #[test]
14616    fn client_builder_rejects_the_sdk_owned_api_suffix() {
14617        for base_url in [
14618            "http://127.0.0.1:8080/api",
14619            "http://localhost:8080/api/",
14620            "https://runtime.example.test/namespaces/orders/api",
14621        ] {
14622            let error = Client::builder(base_url)
14623                .build()
14624                .expect_err("SDK-owned /api suffix must be rejected during build");
14625
14626            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14627            assert!(
14628                error.to_string().contains("SDK appends /api automatically"),
14629                "the validation error must explain how to fix the endpoint"
14630            );
14631        }
14632    }
14633
14634    #[test]
14635    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14636        for (base_url, expected) in [
14637            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14638            (
14639                "http://localhost:8080/durable-workflow/",
14640                "http://localhost:8080/durable-workflow",
14641            ),
14642            (
14643                "https://runtime.example.test/namespaces/orders",
14644                "https://runtime.example.test/namespaces/orders",
14645            ),
14646            (
14647                "https://runtime.example.test/gateway/api/namespaces/orders",
14648                "https://runtime.example.test/gateway/api/namespaces/orders",
14649            ),
14650            (
14651                "https://api.example.test/runtime/orders/",
14652                "https://api.example.test/runtime/orders",
14653            ),
14654        ] {
14655            let client = Client::builder(base_url)
14656                .build()
14657                .expect("Server and Cloud runtime base URL must remain valid");
14658
14659            assert_eq!(client.base_url, expected);
14660        }
14661    }
14662
14663    #[test]
14664    fn workflow_completion_uses_the_additive_command_protocol_floor() {
14665        assert_eq!(
14666            workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14667            WORKER_PROTOCOL_VERSION
14668        );
14669        assert_eq!(
14670            workflow_completion_protocol_version(&[json!({
14671                "type": "upsert_search_attributes",
14672                "attributes": {"OrderStatus": "waiting"},
14673            })]),
14674            SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14675        );
14676        assert_eq!(
14677            workflow_completion_protocol_version(&[json!({
14678                "type": "upsert_search_attributes",
14679                "attributes": {"OrderStatus": "waiting"},
14680                "attribute_types": {"OrderStatus": "keyword"},
14681            })]),
14682            TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14683        );
14684        assert_eq!(
14685            workflow_completion_protocol_version(&[
14686                json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14687                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14688            ]),
14689            MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14690        );
14691        assert_eq!(
14692            workflow_completion_protocol_version(&[
14693                json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14694                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14695            ]),
14696            CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14697        );
14698        assert_eq!(
14699            workflow_completion_protocol_version(&[json!({
14700                "type": "open_condition_wait",
14701                "condition_wait_occurrence_id": "rust:condition-wait:0",
14702                "condition_key": "ready",
14703            })]),
14704            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14705        );
14706        assert_eq!(
14707            workflow_completion_protocol_version_with_message_streams(
14708                &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14709                true,
14710            ),
14711            MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14712        );
14713        assert_eq!(
14714            workflow_completion_protocol_version_with_message_streams(
14715                &[json!({
14716                    "type": "open_condition_wait",
14717                    "condition_wait_occurrence_id": "rust:condition-wait:0",
14718                    "condition_key": "ready",
14719                })],
14720                true,
14721            ),
14722            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14723        );
14724    }
14725
14726    #[test]
14727    fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14728        let manifest = portable_worker_affinity_capability_manifest();
14729
14730        for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14731            assert_eq!(manifest[capability]["supported"], json!(false));
14732            assert_eq!(
14733                manifest[capability]["minimum_protocol_version"],
14734                json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14735            );
14736            assert!(manifest[capability]["reason"]
14737                .as_str()
14738                .is_some_and(|reason| !reason.is_empty()));
14739        }
14740    }
14741
14742    fn typed_fidelity_probe() -> AvroValue {
14743        AvroValue::Map(BTreeMap::from([
14744            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
14745            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
14746            (
14747                "numeric".to_string(),
14748                AvroValue::Map(BTreeMap::from([
14749                    ("0".to_string(), AvroValue::String("zero".to_string())),
14750                    ("1".to_string(), AvroValue::String("one".to_string())),
14751                ])),
14752            ),
14753            (
14754                "nested".to_string(),
14755                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
14756                    "enabled".to_string(),
14757                    AvroValue::Boolean(true),
14758                )]))]),
14759            ),
14760            (
14761                "projection_collisions".to_string(),
14762                AvroValue::Array(projection_collision_probe()),
14763            ),
14764        ]))
14765    }
14766
14767    fn projection_collision_probe() -> Vec<AvroValue> {
14768        vec![
14769            AvroValue::Map(BTreeMap::from([
14770                ("$type".to_string(), AvroValue::String("bytes".to_string())),
14771                (
14772                    "base64".to_string(),
14773                    AvroValue::String("ordinary user text".to_string()),
14774                ),
14775            ])),
14776            AvroValue::Map(BTreeMap::from([
14777                ("$type".to_string(), AvroValue::String("map".to_string())),
14778                (
14779                    "entries".to_string(),
14780                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
14781                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
14782                        (
14783                            "value".to_string(),
14784                            AvroValue::String("user map".to_string()),
14785                        ),
14786                    ]))]),
14787                ),
14788            ])),
14789        ]
14790    }
14791
14792    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14793    struct TypedContract {
14794        nested: TypedNested,
14795        mode: TypedMode,
14796        optional: Option<String>,
14797        absent: Option<String>,
14798        items: Vec<i64>,
14799        labels: BTreeMap<String, String>,
14800        bytes: serde_bytes::ByteBuf,
14801        signed: i64,
14802        finite: f64,
14803    }
14804
14805    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14806    struct TypedNested {
14807        enabled: bool,
14808    }
14809
14810    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14811    enum TypedMode {
14812        Detailed { label: String },
14813    }
14814
14815    fn typed_contract() -> TypedContract {
14816        TypedContract {
14817            nested: TypedNested { enabled: true },
14818            mode: TypedMode::Detailed {
14819                label: "compiler-checked".to_string(),
14820            },
14821            optional: Some("present".to_string()),
14822            absent: None,
14823            items: vec![i64::MIN, 0, i64::MAX],
14824            labels: BTreeMap::from([
14825                ("language".to_string(), "rust".to_string()),
14826                ("wire".to_string(), "avro".to_string()),
14827            ]),
14828            bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
14829            signed: -9_223_372_036_854_775_000,
14830            finite: 12.5,
14831        }
14832    }
14833
14834    #[derive(Clone, Debug, Default, PartialEq)]
14835    struct ReplayCounterState {
14836        loaded: Option<String>,
14837        count: i64,
14838        finished: bool,
14839    }
14840
14841    fn replay_counter_worker() -> Worker {
14842        let client = Client::new("http://127.0.0.1:8080").expect("client");
14843        let mut worker = Worker::new(client, "rust-workers");
14844        worker.register_replayed_workflow(
14845            "replay-counter",
14846            ReplayCounterState::default,
14847            |ctx, _input, state| async move {
14848                let loaded = ctx.activity("load-counter", json!([])).await?;
14849                state.update(|current| {
14850                    current.loaded = loaded.as_str().map(str::to_string);
14851                })?;
14852                for _ in 0..2 {
14853                    let signal = ctx.wait_signal("increment").await?;
14854                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
14855                    state.update(|current| current.count += amount)?;
14856                }
14857                state.update(|current| current.finished = true)?;
14858                state.read(|current| Ok(json!(current.count)))?
14859            },
14860        );
14861        worker.register_replayed_query::<ReplayCounterState, _, _>(
14862            "replay-counter",
14863            "current",
14864            |_ctx, state, _args| async move {
14865                Ok(json!({
14866                    "loaded": state.loaded,
14867                    "count": state.count,
14868                    "finished": state.finished,
14869                }))
14870            },
14871        );
14872        worker.register_replayed_query::<ReplayCounterState, _, _>(
14873            "replay-counter",
14874            "detached-mutation",
14875            |_ctx, state, _args| async move {
14876                let mut detached = (*state).clone();
14877                detached.count = 999;
14878                Ok(json!(detached.count))
14879            },
14880        );
14881        worker.register_replayed_query::<ReplayCounterState, _, _>(
14882            "replay-counter",
14883            "failed-mutation",
14884            |_ctx, state, _args| async move {
14885                let mut detached = (*state).clone();
14886                detached.count = 999;
14887                Err(Error::WorkerLoop("query refused".to_string()))
14888            },
14889        );
14890        worker
14891    }
14892
14893    fn replay_counter_query(
14894        query_name: &str,
14895        history_events: Value,
14896        run_status: &str,
14897    ) -> QueryTask {
14898        let arguments = fixture_envelope(json!([]));
14899        serde_json::from_value(json!({
14900            "query_task_id": format!("query-{query_name}"),
14901            "workflow_type": "replay-counter",
14902            "query_name": query_name,
14903            "payload_codec": DEFAULT_CODEC,
14904            "workflow_arguments": arguments.clone(),
14905            "query_arguments": arguments,
14906            "history_events": history_events,
14907            "run_status": run_status,
14908        }))
14909        .expect("query task")
14910    }
14911
14912    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
14913        workflow_context_with_codec(history, DEFAULT_CODEC)
14914    }
14915
14916    fn workflow_context_with_codec(
14917        history: Vec<HistoryEvent>,
14918        payload_codec: &str,
14919    ) -> WorkflowContext {
14920        WorkflowContext {
14921            state: Arc::new(Mutex::new(
14922                WorkflowState::new_with_identity(
14923                    history,
14924                    None,
14925                    None,
14926                    "rust-workers".to_string(),
14927                    payload_codec.to_string(),
14928                    None,
14929                )
14930                .expect("valid workflow history"),
14931            )),
14932        }
14933    }
14934
14935    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
14936        HistoryEvent {
14937            event_type: event_type.to_string(),
14938            payload,
14939            raw: HashMap::new(),
14940        }
14941    }
14942
14943    fn parallel_path_entry(
14944        kind: &str,
14945        base: u64,
14946        size: usize,
14947        index: usize,
14948    ) -> ParallelGroupMetadata {
14949        parallel_group_entry(base, size, index, kind)
14950    }
14951
14952    fn parallel_history_event(
14953        event_type: &str,
14954        sequence: u64,
14955        identity_field: &str,
14956        identity: &str,
14957        path: Vec<ParallelGroupMetadata>,
14958        result: Option<Value>,
14959    ) -> HistoryEvent {
14960        let mut payload = serde_json::Map::from_iter([
14961            ("sequence".to_string(), json!(sequence)),
14962            (identity_field.to_string(), json!(identity)),
14963        ]);
14964        let inner = path.last().expect("parallel history path");
14965        apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
14966        payload.insert("parallel_group_path".to_string(), json!(path));
14967        if let Some(result) = result {
14968            let field = if event_type == "ChildRunCompleted" {
14969                "result"
14970            } else {
14971                "result"
14972            };
14973            payload.insert(field.to_string(), fixture_envelope(result));
14974            payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
14975        }
14976        history_event(event_type, Value::Object(payload))
14977    }
14978
14979    fn nested_parallel_operations() -> Vec<ParallelOperation> {
14980        vec![
14981            ParallelOperation::activity("first", json!([])),
14982            ParallelOperation::group(vec![
14983                ParallelOperation::child_workflow(
14984                    "second",
14985                    ChildWorkflowOptions::new("child-workers"),
14986                    json!([]),
14987                ),
14988                ParallelOperation::activity("third", json!([])),
14989            ]),
14990        ]
14991    }
14992
14993    fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
14994        let outer = [
14995            parallel_path_entry("mixed", 1, 3, 0),
14996            parallel_path_entry("mixed", 1, 3, 1),
14997            parallel_path_entry("mixed", 1, 3, 2),
14998        ];
14999        [
15000            vec![outer[0].clone()],
15001            vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15002            vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15003        ]
15004    }
15005
15006    #[test]
15007    fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15008        let ctx = workflow_context(Vec::new());
15009        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15010        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15011
15012        assert!(matches!(
15013            call.as_mut().poll(&mut task_context),
15014            Poll::Pending
15015        ));
15016        let commands = ctx.take_commands().expect("parallel commands");
15017        assert_eq!(
15018            commands
15019                .iter()
15020                .map(|command| command["type"].as_str().unwrap_or_default())
15021                .collect::<Vec<_>>(),
15022            [
15023                "schedule_activity",
15024                "start_child_workflow",
15025                "schedule_activity"
15026            ]
15027        );
15028        let paths = nested_parallel_paths();
15029        for (command, path) in commands.iter().zip(paths) {
15030            assert_eq!(command["parallel_group_path"], json!(path));
15031            assert_eq!(
15032                command["parallel_group_id"],
15033                json!(path.last().expect("inner group").parallel_group_id)
15034            );
15035        }
15036    }
15037
15038    fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15039        let paths = nested_parallel_paths();
15040        let third = parallel_history_event(
15041            "ActivityCompleted",
15042            3,
15043            "activity_type",
15044            "third",
15045            paths[2].clone(),
15046            Some(json!("three")),
15047        );
15048        vec![
15049            parallel_history_event(
15050                "ActivityCompleted",
15051                1,
15052                "activity_type",
15053                "first",
15054                paths[0].clone(),
15055                Some(json!("one")),
15056            ),
15057            parallel_history_event(
15058                "ChildWorkflowScheduled",
15059                2,
15060                "child_workflow_type",
15061                "second",
15062                paths[1].clone(),
15063                None,
15064            ),
15065            parallel_history_event(
15066                "ChildRunCompleted",
15067                2,
15068                "child_workflow_type",
15069                "second",
15070                paths[1].clone(),
15071                Some(json!("two")),
15072            ),
15073            third.clone(),
15074            third,
15075        ]
15076    }
15077
15078    #[test]
15079    fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15080        for _restart_or_completed_replay in 0..2 {
15081            let ctx = workflow_context(completed_nested_parallel_history());
15082            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15083            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15084            let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15085                panic!("completed nested parallel history must replay");
15086            };
15087            assert_eq!(
15088                results,
15089                vec![
15090                    ParallelResult::Activity(json!("one")),
15091                    ParallelResult::Group(vec![
15092                        ParallelResult::ChildWorkflow(ChildWorkflowResult {
15093                            parent: WorkflowIdentity {
15094                                workflow_id: None,
15095                                run_id: None,
15096                            },
15097                            child: WorkflowIdentity {
15098                                workflow_id: None,
15099                                run_id: None,
15100                            },
15101                            child_workflow_type: Some("second".to_string()),
15102                            result: json!("two"),
15103                        }),
15104                        ParallelResult::Activity(json!("three")),
15105                    ]),
15106                ]
15107            );
15108            assert!(ctx.take_commands().expect("commands").is_empty());
15109            ctx.ensure_history_consumed().expect("history consumed");
15110        }
15111    }
15112
15113    #[test]
15114    fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15115        let paths = nested_parallel_paths();
15116        let history = vec![
15117            parallel_history_event(
15118                "ActivityCompleted",
15119                1,
15120                "activity_type",
15121                "first",
15122                paths[0].clone(),
15123                Some(json!("one")),
15124            ),
15125            parallel_history_event(
15126                "ChildWorkflowScheduled",
15127                2,
15128                "child_workflow_type",
15129                "second",
15130                paths[1].clone(),
15131                None,
15132            ),
15133            parallel_history_event(
15134                "ChildRunFailed",
15135                2,
15136                "child_workflow_type",
15137                "second",
15138                paths[1].clone(),
15139                None,
15140            ),
15141            parallel_history_event(
15142                "ActivityCompleted",
15143                3,
15144                "activity_type",
15145                "third",
15146                paths[2].clone(),
15147                Some(json!("late")),
15148            ),
15149        ];
15150        let ctx = workflow_context(history);
15151        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15152        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15153        let outcome = call.as_mut().poll(&mut task_context);
15154        let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15155            panic!("one failed child must return a typed partial failure: {outcome:?}");
15156        };
15157        assert_eq!(failure.member_path, [1, 0]);
15158        assert_eq!(failure.group_id, "parallel-calls:1:3");
15159        assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15160        assert_eq!(
15161            failure
15162                .completed
15163                .iter()
15164                .map(|completion| completion.member_path.clone())
15165                .collect::<Vec<_>>(),
15166            [vec![0], vec![1, 1]]
15167        );
15168    }
15169
15170    #[test]
15171    fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15172        let paths = nested_parallel_paths();
15173        let history = vec![
15174            parallel_history_event(
15175                "ActivityScheduled",
15176                1,
15177                "activity_type",
15178                "first",
15179                paths[0].clone(),
15180                None,
15181            ),
15182            parallel_history_event(
15183                "ChildWorkflowScheduled",
15184                2,
15185                "child_workflow_type",
15186                "second",
15187                paths[1].clone(),
15188                None,
15189            ),
15190            parallel_history_event(
15191                "ActivityScheduled",
15192                3,
15193                "activity_type",
15194                "third",
15195                paths[2].clone(),
15196                None,
15197            ),
15198        ];
15199        for _restart in 0..2 {
15200            let ctx = workflow_context(history.clone());
15201            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15202            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15203            let outcome = call.as_mut().poll(&mut task_context);
15204            assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15205            assert!(ctx.take_commands().expect("commands").is_empty());
15206        }
15207    }
15208
15209    fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15210        vec![selection_group_entry(
15211            1,
15212            2,
15213            index,
15214            "activity",
15215            &SelectionMemberMetadata {
15216                key: SelectionKey::Name(key.to_string()),
15217                index,
15218                base_sequence: index as u64 + 1,
15219                size: 1,
15220                kind: "activity".to_string(),
15221            },
15222        )]
15223    }
15224
15225    fn selection_activity_event(
15226        event_type: &str,
15227        index: usize,
15228        key: &str,
15229        result: Option<Value>,
15230    ) -> HistoryEvent {
15231        let sequence = index as u64 + 1;
15232        let mut event = parallel_history_event(
15233            event_type,
15234            sequence,
15235            "activity_type",
15236            &format!("{key}-activity"),
15237            selection_path(index, key),
15238            result,
15239        );
15240        event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15241        event.raw.insert(
15242            "id".to_string(),
15243            json!(if event_type == "ActivityCompleted" {
15244                format!("event-{key}")
15245            } else {
15246                format!("{event_type}-{key}")
15247            }),
15248        );
15249        event
15250    }
15251
15252    fn selection_winner_marker() -> HistoryEvent {
15253        history_event(
15254            "SelectionResolved",
15255            json!({
15256                "selection_group_id": "select-calls:1:2",
15257                "selection_group_base_sequence": 1,
15258                "selection_group_size": 2,
15259                "member_key": "fast",
15260                "member_index": 1,
15261                "member_base_sequence": 2,
15262                "member_size": 1,
15263                "operation_kind": "activity",
15264                "operation_identity": "activity-fast",
15265                "outcome": "completed",
15266                "resolution_event_id": "event-fast",
15267                "resolution_event_type": "ActivityCompleted",
15268            }),
15269        )
15270    }
15271
15272    fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15273        ctx.select_keyed(vec![
15274            (
15275                "slow",
15276                ParallelOperation::activity_with_options(
15277                    "slow-activity",
15278                    ActivityOptions::new().task_queue("default"),
15279                    json!([]),
15280                ),
15281            ),
15282            (
15283                "fast",
15284                ParallelOperation::activity_with_options(
15285                    "fast-activity",
15286                    ActivityOptions::new().task_queue("default"),
15287                    json!([]),
15288                ),
15289            ),
15290        ])
15291    }
15292
15293    fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15294        let ctx = workflow_context(history);
15295        let mut call = Box::pin(keyed_activity_selection(&ctx));
15296        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15297        let selected = match call.as_mut().poll(&mut task_context) {
15298            Poll::Ready(Ok(selected)) => selected,
15299            Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15300            Poll::Pending => panic!("persisted selection winner must replay without pending"),
15301        };
15302        assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15303        assert_eq!(
15304            selected.value,
15305            Some(ParallelResult::Activity(json!("winner-value")))
15306        );
15307        let slow = selected
15308            .handle(&SelectionKey::Name("slow".to_string()))
15309            .expect("slow handle")
15310            .clone();
15311        let mut await_slow = Box::pin(slow.await_result());
15312        assert!(matches!(
15313            await_slow.as_mut().poll(&mut task_context),
15314            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15315        ));
15316        assert!(ctx.take_commands().expect("commands").is_empty());
15317    }
15318
15319    const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15320
15321    fn canonical_selection_history() -> Vec<HistoryEvent> {
15322        const FIXTURE: &[u8] =
15323            include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15324        assert_eq!(
15325            format!("{:x}", Sha256::digest(FIXTURE)),
15326            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15327        );
15328        let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15329
15330        serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15331    }
15332
15333    #[test]
15334    fn selection_fresh_process_entrypoint() {
15335        let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15336            return;
15337        };
15338        let persisted = fs::read(path).expect("persisted selection history");
15339        assert_eq!(
15340            format!("{:x}", Sha256::digest(&persisted)),
15341            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15342        );
15343        let fixture: Value =
15344            serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15345        let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15346            .expect("valid persisted selection history");
15347
15348        assert_persisted_selection_replay(history);
15349    }
15350
15351    #[test]
15352    fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15353        let ctx = workflow_context(Vec::new());
15354        let mut call = Box::pin(keyed_activity_selection(&ctx));
15355        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15356
15357        assert!(matches!(
15358            call.as_mut().poll(&mut task_context),
15359            Poll::Pending
15360        ));
15361        let commands = ctx.take_commands().expect("selection commands");
15362        assert_eq!(commands.len(), 2);
15363        assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15364        assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15365        assert!(commands.iter().all(|command| {
15366            command["parallel_group_id"] == json!("select-calls:1:2")
15367                && command["parallel_group_mode"] == json!("select")
15368        }));
15369    }
15370
15371    #[test]
15372    fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15373        let ctx = workflow_context(Vec::new());
15374        let mut invalid = Box::pin(ctx.select_keyed(vec![(
15375            "",
15376            ParallelOperation::activity("invalid", json!([])),
15377        )]));
15378        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15379        assert!(matches!(
15380            invalid.as_mut().poll(&mut task_context),
15381            Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15382                reason: "selection_key_invalid",
15383                ..
15384            })))
15385        ));
15386
15387        for invalid_key in [json!(""), json!(-1)] {
15388            let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15389            event.payload["selection_member_key"] = invalid_key.clone();
15390            event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15391            assert!(matches!(
15392                WorkflowState::new_with_identity(
15393                    vec![event],
15394                    None,
15395                    None,
15396                    "rust-workers".to_string(),
15397                    DEFAULT_CODEC.to_string(),
15398                    None,
15399                ),
15400                Err(Error::NonDeterministicReplay(_))
15401            ));
15402        }
15403    }
15404
15405    #[test]
15406    fn selection_preserves_valid_named_and_numeric_keys() {
15407        let ctx = workflow_context(Vec::new());
15408        let mut selection = Box::pin(ctx.select_keyed(vec![
15409            (
15410                SelectionKey::Index(0),
15411                ParallelOperation::activity("numeric", json!([])),
15412            ),
15413            (
15414                SelectionKey::Name("named".to_string()),
15415                ParallelOperation::timer(Duration::from_secs(1)),
15416            ),
15417        ]));
15418        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15419
15420        assert!(matches!(
15421            selection.as_mut().poll(&mut task_context),
15422            Poll::Pending
15423        ));
15424        let commands = ctx.take_commands().expect("selection commands");
15425        assert_eq!(commands[0]["selection_member_key"], json!(0));
15426        assert_eq!(commands[1]["selection_member_key"], json!("named"));
15427    }
15428
15429    #[test]
15430    fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15431        let history = canonical_selection_history();
15432        assert_persisted_selection_replay(history.clone());
15433
15434        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15435            .join("tests/fixtures/durable_selection_runtime_history.json");
15436        let output =
15437            ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15438                .args([
15439                    "--exact",
15440                    "tests::selection_fresh_process_entrypoint",
15441                    "--nocapture",
15442                ])
15443                .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15444                .output()
15445                .expect("run fresh selection replay process");
15446
15447        assert!(
15448            output.status.success(),
15449            "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15450            String::from_utf8_lossy(&output.stdout),
15451            String::from_utf8_lossy(&output.stderr),
15452        );
15453    }
15454
15455    #[test]
15456    fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15457        let mut history = canonical_selection_history();
15458        history.retain(|event| event.event_type != "SelectionResolved");
15459        let ctx = workflow_context(history);
15460        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15461        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15462
15463        assert!(matches!(
15464            selection.as_mut().poll(&mut task_context),
15465            Poll::Pending
15466        ));
15467        assert!(ctx.take_commands().expect("commands").is_empty());
15468        assert!(
15469            ctx.matched_recorded_pending()
15470                .expect("selection pending state"),
15471            "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15472        );
15473    }
15474
15475    #[test]
15476    fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15477        for (terminal_event, predicate_satisfied, timeout_seconds) in [
15478            ("ConditionWaitSatisfied", true, None),
15479            ("ConditionWaitTimedOut", false, Some(0)),
15480        ] {
15481            let member = SelectionMemberMetadata {
15482                key: SelectionKey::Name("condition".to_string()),
15483                index: 0,
15484                base_sequence: 1,
15485                size: 1,
15486                kind: "condition".to_string(),
15487            };
15488            let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15489            let mut payload = json!({
15490                "sequence": 1,
15491                "condition_wait_id": "condition-1",
15492                "condition_wait_occurrence_id": "rust:condition-wait:0",
15493                "condition_key": "ready",
15494                "condition_definition_fingerprint": "sha256:ready-v1",
15495                "parallel_group_path": path,
15496            });
15497            payload
15498                .as_object_mut()
15499                .expect("condition history payload")
15500                .extend(
15501                    serde_json::to_value(&path[0])
15502                        .expect("condition selection metadata")
15503                        .as_object()
15504                        .expect("condition selection metadata object")
15505                        .clone(),
15506                );
15507            if let Some(timeout_seconds) = timeout_seconds {
15508                payload["timeout_seconds"] = json!(timeout_seconds);
15509            }
15510            let history = vec![
15511                history_event("ConditionWaitOpened", payload.clone()),
15512                history_event(terminal_event, payload),
15513            ];
15514            let ctx = workflow_context(history);
15515            let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15516            if timeout_seconds.is_some() {
15517                options = options.timeout(Duration::ZERO);
15518            }
15519            let mut selection = Box::pin(ctx.select_keyed(vec![(
15520                "condition",
15521                ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15522            )]));
15523            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15524
15525            assert!(matches!(
15526                selection.as_mut().poll(&mut task_context),
15527                Poll::Pending
15528            ));
15529            assert!(ctx.take_commands().expect("commands").is_empty());
15530            assert!(
15531                ctx.matched_recorded_pending()
15532                    .expect("condition selection pending state"),
15533                "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15534            );
15535        }
15536    }
15537
15538    #[test]
15539    fn selection_immediate_condition_members_open_a_durable_wait() {
15540        for predicate_satisfied in [true, false] {
15541            let ctx = workflow_context(Vec::new());
15542            let mut selection = Box::pin(ctx.select_keyed(vec![(
15543                "condition",
15544                ParallelOperation::condition(
15545                    ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15546                    move || Ok(predicate_satisfied),
15547                ),
15548            )]));
15549            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15550
15551            assert!(matches!(
15552                selection.as_mut().poll(&mut task_context),
15553                Poll::Pending
15554            ));
15555            let commands = ctx.take_commands().expect("condition selection command");
15556            assert_eq!(commands.len(), 1);
15557            assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15558            assert_eq!(commands[0]["timeout_seconds"], json!(0));
15559            assert_eq!(
15560                commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15561                json!("select")
15562            );
15563        }
15564    }
15565
15566    #[test]
15567    fn selection_loser_cancellation_is_explicit_and_idempotent() {
15568        let history = vec![
15569            selection_activity_event("ActivityScheduled", 0, "slow", None),
15570            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15571            selection_winner_marker(),
15572        ];
15573        let ctx = workflow_context(history.clone());
15574        let mut call = Box::pin(keyed_activity_selection(&ctx));
15575        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15576        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15577            panic!("winner must replay");
15578        };
15579        let slow = selected
15580            .handle(&SelectionKey::Name("slow".to_string()))
15581            .expect("slow handle")
15582            .clone();
15583        let mut cancel = Box::pin(slow.cancel());
15584        assert!(matches!(
15585            cancel.as_mut().poll(&mut task_context),
15586            Poll::Pending
15587        ));
15588        assert!(matches!(
15589            cancel.as_mut().poll(&mut task_context),
15590            Poll::Pending
15591        ));
15592        let commands = ctx.take_commands().expect("cancel command");
15593        assert_eq!(commands.len(), 1);
15594        assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15595        assert_eq!(commands[0]["member_key"], json!("slow"));
15596
15597        let mut cancelled_history = history;
15598        cancelled_history.push(history_event(
15599            "SelectionOperationCancelled",
15600            json!({
15601                "selection_group_id": "select-calls:1:2",
15602                "member_key": "slow",
15603                "member_index": 0,
15604                "member_base_sequence": 1,
15605                "member_size": 1,
15606                "operation_kind": "activity",
15607                "operation_identity": "activity-slow",
15608                "cancelled_at": "2026-08-27T00:00:00Z",
15609            }),
15610        ));
15611        let replayed = workflow_context(cancelled_history);
15612        let mut call = Box::pin(keyed_activity_selection(&replayed));
15613        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15614            panic!("winner must replay after cancellation");
15615        };
15616        let slow = selected
15617            .handle(&SelectionKey::Name("slow".to_string()))
15618            .expect("slow handle")
15619            .clone();
15620        let mut cancel = Box::pin(slow.cancel());
15621        assert!(matches!(
15622            cancel.as_mut().poll(&mut task_context),
15623            Poll::Ready(Ok(()))
15624        ));
15625        assert!(replayed.take_commands().expect("commands").is_empty());
15626    }
15627
15628    #[test]
15629    fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15630        let base_history = vec![
15631            selection_activity_event("ActivityScheduled", 0, "slow", None),
15632            selection_activity_event("ActivityScheduled", 1, "fast", None),
15633            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15634            selection_winner_marker(),
15635        ];
15636        for (field, corrupt) in [
15637            ("member_key", json!("fast")),
15638            ("member_index", json!(1)),
15639            ("member_base_sequence", json!(3)),
15640            ("member_size", json!(2)),
15641            ("operation_kind", json!("timer")),
15642            ("operation_identity", json!("forged")),
15643        ] {
15644            let mut cancellation = json!({
15645                "selection_group_id": "select-calls:1:2",
15646                "member_key": "slow",
15647                "member_index": 0,
15648                "member_base_sequence": 1,
15649                "member_size": 1,
15650                "operation_kind": "activity",
15651                "operation_identity": "activity-slow",
15652            });
15653            cancellation[field] = corrupt;
15654            let mut history = base_history.clone();
15655            history.push(history_event("SelectionOperationCancelled", cancellation));
15656            let ctx = workflow_context(history);
15657            let mut selection = Box::pin(keyed_activity_selection(&ctx));
15658            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15659
15660            assert!(matches!(
15661                selection.as_mut().poll(&mut task_context),
15662                Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15663            ));
15664        }
15665    }
15666
15667    #[test]
15668    fn selection_child_identity_prefers_the_durable_run_id() {
15669        let ctx = workflow_context(vec![history_event(
15670            "ChildWorkflowScheduled",
15671            json!({
15672                "sequence": 1,
15673                "child_workflow_type": "child",
15674                "child_workflow_instance_id": "child-instance",
15675                "child_workflow_run_id": "child-run",
15676            }),
15677        )]);
15678        let state = ctx.state.lock().expect("workflow state");
15679
15680        assert_eq!(
15681            selection_operation_identity(&state, "child", 1, 1),
15682            "child-run"
15683        );
15684    }
15685
15686    #[test]
15687    fn selection_activity_identity_requires_canonical_execution_id() {
15688        let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15689        let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15690        let mut fast_completed =
15691            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15692        for event in [&mut fast_open, &mut fast_completed] {
15693            event
15694                .payload
15695                .as_object_mut()
15696                .expect("activity payload")
15697                .remove("activity_execution_id");
15698            event.payload["activity_id"] = json!("forged-activity-id");
15699        }
15700        let mut marker = selection_winner_marker();
15701        marker.payload["operation_identity"] = json!("forged-activity-id");
15702        let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15703        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15704        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15705
15706        assert!(matches!(
15707            selection.as_mut().poll(&mut task_context),
15708            Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15709        ));
15710    }
15711
15712    #[test]
15713    fn selection_completion_before_cancellation_remains_awaitable() {
15714        let history = vec![
15715            selection_activity_event("ActivityScheduled", 0, "slow", None),
15716            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15717            selection_winner_marker(),
15718            selection_activity_event(
15719                "ActivityCompleted",
15720                0,
15721                "slow",
15722                Some(json!("completed-first")),
15723            ),
15724        ];
15725        let ctx = workflow_context(history);
15726        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15727        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15728        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15729            panic!("winner must replay");
15730        };
15731        let slow = selected
15732            .handle(&SelectionKey::Name("slow".to_string()))
15733            .expect("slow handle")
15734            .clone();
15735        let mut cancel = Box::pin(slow.cancel());
15736        assert!(matches!(
15737            cancel.as_mut().poll(&mut task_context),
15738            Poll::Ready(Ok(()))
15739        ));
15740        let mut await_slow = Box::pin(slow.await_result());
15741        assert!(matches!(
15742            await_slow.as_mut().poll(&mut task_context),
15743            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
15744        ));
15745        let commands = ctx.take_commands().expect("commands");
15746        assert!(commands.is_empty());
15747    }
15748
15749    #[test]
15750    fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
15751        let nested_member = SelectionMemberMetadata {
15752            key: SelectionKey::Name("nested".to_string()),
15753            index: 0,
15754            base_sequence: 1,
15755            size: 2,
15756            kind: "group".to_string(),
15757        };
15758        let deadline_member = SelectionMemberMetadata {
15759            key: SelectionKey::Name("deadline".to_string()),
15760            index: 1,
15761            base_sequence: 3,
15762            size: 1,
15763            kind: "timer".to_string(),
15764        };
15765        let nested_paths = [
15766            vec![
15767                selection_group_entry(1, 3, 0, "mixed", &nested_member),
15768                parallel_group_entry(1, 2, 0, "activity"),
15769            ],
15770            vec![
15771                selection_group_entry(1, 3, 1, "mixed", &nested_member),
15772                parallel_group_entry(1, 2, 1, "activity"),
15773            ],
15774        ];
15775        let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
15776        let mut timer_fired = parallel_history_event(
15777            "TimerFired",
15778            3,
15779            "timer_id",
15780            "timer-3",
15781            deadline_path.clone(),
15782            None,
15783        );
15784        timer_fired.payload["delay_seconds"] = json!(0);
15785        timer_fired
15786            .raw
15787            .insert("id".to_string(), json!("timer-fired"));
15788        let mut timer_scheduled = parallel_history_event(
15789            "TimerScheduled",
15790            3,
15791            "timer_id",
15792            "timer-3",
15793            deadline_path,
15794            None,
15795        );
15796        timer_scheduled.payload["delay_seconds"] = json!(0);
15797        let history = vec![
15798            parallel_history_event(
15799                "ActivityScheduled",
15800                1,
15801                "activity_type",
15802                "nested-first",
15803                nested_paths[0].clone(),
15804                None,
15805            ),
15806            parallel_history_event(
15807                "ActivityScheduled",
15808                2,
15809                "activity_type",
15810                "nested-second",
15811                nested_paths[1].clone(),
15812                None,
15813            ),
15814            timer_scheduled,
15815            timer_fired,
15816            history_event(
15817                "SelectionResolved",
15818                json!({
15819                    "selection_group_id": "select-calls:1:3",
15820                    "selection_group_base_sequence": 1,
15821                    "selection_group_size": 3,
15822                    "member_key": "deadline",
15823                    "member_index": 1,
15824                    "member_base_sequence": 3,
15825                    "member_size": 1,
15826                    "operation_kind": "timer",
15827                    "operation_identity": "timer-3",
15828                    "outcome": "completed",
15829                    "resolution_event_id": "timer-fired",
15830                    "resolution_event_type": "TimerFired",
15831                }),
15832            ),
15833            parallel_history_event(
15834                "ActivityFailed",
15835                2,
15836                "activity_type",
15837                "nested-second",
15838                nested_paths[1].clone(),
15839                None,
15840            ),
15841        ];
15842        let ctx = workflow_context(history);
15843        let mut selection = Box::pin(ctx.select_keyed(vec![
15844            (
15845                "nested",
15846                ParallelOperation::group(vec![
15847                    ParallelOperation::activity("nested-first", json!([])),
15848                    ParallelOperation::activity("nested-second", json!([])),
15849                ]),
15850            ),
15851            ("deadline", ParallelOperation::timer(Duration::ZERO)),
15852        ]));
15853        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15854        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15855            panic!("deadline winner must replay");
15856        };
15857        let nested = selected
15858            .handle(&SelectionKey::Name("nested".to_string()))
15859            .expect("nested handle")
15860            .clone();
15861        let mut cancel = Box::pin(nested.cancel());
15862        assert!(matches!(
15863            cancel.as_mut().poll(&mut task_context),
15864            Poll::Ready(Ok(()))
15865        ));
15866        let mut await_nested = Box::pin(nested.await_result());
15867
15868        assert!(matches!(
15869            await_nested.as_mut().poll(&mut task_context),
15870            Poll::Ready(Err(Error::ActivityFailed(_)))
15871        ));
15872        assert!(ctx.take_commands().expect("commands").is_empty());
15873    }
15874
15875    #[test]
15876    fn selection_supports_child_timer_signal_condition_and_nested_groups() {
15877        let ctx = workflow_context(Vec::new());
15878        let mut call = Box::pin(ctx.select(vec![
15879            ParallelOperation::child_workflow(
15880                "child",
15881                ChildWorkflowOptions::new("children"),
15882                json!([]),
15883            ),
15884            ParallelOperation::timer(Duration::from_secs(30)),
15885            ParallelOperation::signal("approval"),
15886            ParallelOperation::condition(
15887                ConditionWaitOptions::new("ready", "sha256:ready"),
15888                || Ok(false),
15889            ),
15890            ParallelOperation::group(vec![
15891                ParallelOperation::activity("nested-one", json!([])),
15892                ParallelOperation::activity("nested-two", json!([])),
15893            ]),
15894        ]));
15895        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15896        assert!(matches!(
15897            call.as_mut().poll(&mut task_context),
15898            Poll::Pending
15899        ));
15900        let commands = ctx.take_commands().expect("selection commands");
15901        assert_eq!(
15902            commands
15903                .iter()
15904                .map(|command| command["type"].as_str().unwrap_or_default())
15905                .collect::<Vec<_>>(),
15906            [
15907                "start_child_workflow",
15908                "start_timer",
15909                "open_signal_wait",
15910                "open_condition_wait",
15911                "schedule_activity",
15912                "schedule_activity",
15913            ]
15914        );
15915        assert!(commands.iter().all(|command| {
15916            command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
15917        }));
15918        assert_eq!(
15919            commands[4]["parallel_group_path"].as_array().map(Vec::len),
15920            Some(2)
15921        );
15922        assert_eq!(
15923            commands[4]["parallel_group_path"][0]["selection_member_kind"],
15924            json!("group")
15925        );
15926        assert_eq!(
15927            commands[5]["parallel_group_path"][0]["selection_member_kind"],
15928            json!("group")
15929        );
15930
15931        let one_leaf_ctx = workflow_context(Vec::new());
15932        let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
15933            ParallelOperation::activity("nested-only", json!([])),
15934        ])]));
15935        assert!(matches!(
15936            one_leaf.as_mut().poll(&mut task_context),
15937            Poll::Pending
15938        ));
15939        let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
15940        assert_eq!(one_leaf_commands.len(), 1);
15941        assert_eq!(
15942            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
15943            json!("group")
15944        );
15945        assert_eq!(
15946            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
15947            json!(1)
15948        );
15949    }
15950
15951    async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
15952        let mut saga = ctx.saga();
15953        let outcome = async {
15954            let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
15955            saga.add_compensation("trip.cancel-flight", json!([flight]))?;
15956            let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
15957            saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
15958            ctx.activity("trip.charge", json!([])).await?;
15959            Ok(json!({"status": "booked"}))
15960        }
15961        .await;
15962        saga.finish(outcome).await
15963    }
15964
15965    fn saga_activity(
15966        event_type: &str,
15967        sequence: u64,
15968        activity_type: &str,
15969        result: Option<Value>,
15970    ) -> HistoryEvent {
15971        let mut payload = json!({
15972            "sequence": sequence,
15973            "activity_type": activity_type,
15974            "message": format!("{activity_type} failed"),
15975            "exception_type": "PlannedFailure",
15976            "non_retryable": true,
15977        });
15978        if let Some(result) = result {
15979            payload["result"] = fixture_envelope(result);
15980        }
15981        history_event(event_type, payload)
15982    }
15983
15984    #[test]
15985    fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
15986        let completed_hotel_compensation = saga_activity(
15987            "ActivityCompleted",
15988            4,
15989            "trip.cancel-hotel",
15990            Some(Value::Null),
15991        );
15992        let history = vec![
15993            saga_activity(
15994                "ActivityCompleted",
15995                1,
15996                "trip.reserve-flight",
15997                Some(json!("flight-1")),
15998            ),
15999            saga_activity(
16000                "ActivityCompleted",
16001                2,
16002                "trip.reserve-hotel",
16003                Some(json!("hotel-1")),
16004            ),
16005            saga_activity("ActivityFailed", 3, "trip.charge", None),
16006            completed_hotel_compensation.clone(),
16007            completed_hotel_compensation,
16008        ];
16009
16010        for _restart in 0..2 {
16011            let ctx = workflow_context(history.clone());
16012            let mut future = Box::pin(trip_saga(ctx.clone()));
16013            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16014            assert!(matches!(
16015                future.as_mut().poll(&mut task_context),
16016                Poll::Pending
16017            ));
16018            let commands = ctx.take_commands().expect("compensation command");
16019            assert_eq!(commands.len(), 1);
16020            assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16021        }
16022    }
16023
16024    #[test]
16025    fn saga_compensation_failure_preserves_both_typed_failures() {
16026        let history = vec![
16027            saga_activity(
16028                "ActivityCompleted",
16029                1,
16030                "trip.reserve-flight",
16031                Some(json!("flight-1")),
16032            ),
16033            saga_activity(
16034                "ActivityCompleted",
16035                2,
16036                "trip.reserve-hotel",
16037                Some(json!("hotel-1")),
16038            ),
16039            saga_activity("ActivityFailed", 3, "trip.charge", None),
16040            saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16041        ];
16042        let ctx = workflow_context(history);
16043        let mut future = Box::pin(trip_saga(ctx));
16044        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16045        let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16046            future.as_mut().poll(&mut task_context)
16047        else {
16048            panic!("compensation failure must remain structured");
16049        };
16050        assert!(matches!(
16051            *failure.initiating_failure,
16052            Error::ActivityFailed(_)
16053        ));
16054        assert!(matches!(
16055            *failure.compensation_failure,
16056            Error::ActivityFailed(_)
16057        ));
16058        assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16059        assert_eq!(failure.compensation_registration_order, 2);
16060    }
16061
16062    #[test]
16063    fn saga_compensates_cooperative_cancellation() {
16064        let ctx = workflow_context(vec![saga_activity(
16065            "ActivityCompleted",
16066            1,
16067            "trip.reserve-flight",
16068            Some(json!("flight-1")),
16069        )]);
16070        ctx.state.lock().expect("state").cancel_requested = true;
16071        let run = {
16072            let ctx = ctx.clone();
16073            async move {
16074                let mut saga = ctx.saga();
16075                let outcome = async {
16076                    let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16077                    saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16078                    ctx.throw_if_cancellation_requested()?;
16079                    Ok(json!("unexpected"))
16080                }
16081                .await;
16082                saga.finish(outcome).await
16083            }
16084        };
16085        let mut future = Box::pin(run);
16086        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16087        assert!(matches!(
16088            future.as_mut().poll(&mut task_context),
16089            Poll::Pending
16090        ));
16091        let commands = ctx.take_commands().expect("cancellation compensation");
16092        assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16093    }
16094
16095    fn workflow_task(
16096        workflow_type: &str,
16097        history_events: Vec<HistoryEvent>,
16098        payload_codec: &str,
16099    ) -> WorkflowTask {
16100        WorkflowTask {
16101            task_id: format!("wft-{workflow_type}"),
16102            workflow_command_id: None,
16103            workflow_id: Some(format!("wf-{workflow_type}")),
16104            run_id: Some(format!("run-{workflow_type}")),
16105            workflow_type: workflow_type.to_string(),
16106            cancel_requested: false,
16107            payload_codec: payload_codec.to_string(),
16108            arguments: Some(
16109                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16110            ),
16111            total_history_events: Some(history_events.len() as u64),
16112            history_size_bytes: None,
16113            continue_as_new_recommended: None,
16114            history_budget_pressure: None,
16115            history_events,
16116            next_history_page_token: None,
16117            workflow_task_attempt: 1,
16118            workflow_signal_id: None,
16119            signal_name: None,
16120            signal_arguments: None,
16121            workflow_update_id: None,
16122            update_name: None,
16123            lease_owner: Some("rust-worker".to_string()),
16124        }
16125    }
16126
16127    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16128    struct SideEffectProbe {
16129        request_id: String,
16130        attempt: u32,
16131    }
16132
16133    #[test]
16134    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16135        let calls = AtomicUsize::new(0);
16136        let ctx = workflow_context(Vec::new());
16137        let value = ctx
16138            .side_effect(|| {
16139                calls.fetch_add(1, Ordering::SeqCst);
16140                SideEffectProbe {
16141                    request_id: "request-42".to_string(),
16142                    attempt: 3,
16143                }
16144            })
16145            .expect("first side effect");
16146        assert_eq!(value.attempt, 3);
16147        assert_eq!(calls.load(Ordering::SeqCst), 1);
16148        let commands = ctx.take_commands().expect("commands");
16149        assert_eq!(commands.len(), 1);
16150        assert_eq!(commands[0]["type"], "record_side_effect");
16151        assert_eq!(
16152            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16153            serde_json::to_value(&value).expect("value")
16154        );
16155
16156        let replay = workflow_context(vec![history_event(
16157            "SideEffectRecorded",
16158            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16159        )]);
16160        let replayed: SideEffectProbe = replay
16161            .side_effect(|| {
16162                calls.fetch_add(1, Ordering::SeqCst);
16163                panic!("committed side-effect callbacks must not run during replay")
16164            })
16165            .expect("replayed side effect");
16166        assert_eq!(replayed, value);
16167        assert_eq!(calls.load(Ordering::SeqCst), 1);
16168        assert!(replay.take_commands().expect("commands").is_empty());
16169        replay.ensure_history_consumed().expect("history consumed");
16170    }
16171
16172    #[test]
16173    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16174        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16175        let value = ctx
16176            .side_effect(|| SideEffectProbe {
16177                request_id: "avro-request".to_string(),
16178                attempt: 1,
16179            })
16180            .expect("Avro side effect");
16181        let uuid = ctx.uuid_v4().expect("deterministic UUID");
16182        let commands = ctx.take_commands().expect("commands");
16183        assert_eq!(commands.len(), 2);
16184        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16185        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16186        assert_eq!(
16187            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16188            serde_json::to_value(&value).expect("value")
16189        );
16190
16191        let replay = workflow_context_with_codec(
16192            vec![
16193                history_event(
16194                    "SideEffectRecorded",
16195                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16196                ),
16197                history_event(
16198                    "SideEffectRecorded",
16199                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16200                ),
16201            ],
16202            DEFAULT_CODEC,
16203        );
16204        let replayed: SideEffectProbe = replay
16205            .side_effect(|| panic!("Avro callback must not run"))
16206            .expect("replayed Avro value");
16207        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16208        assert_eq!(replayed, value);
16209        assert_eq!(replayed_uuid, uuid);
16210        assert!(replay.take_commands().expect("commands").is_empty());
16211    }
16212
16213    #[test]
16214    fn typed_side_effect_replay_preserves_bytes_and_maps() {
16215        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16216        let value = ctx
16217            .side_effect_avro_value(typed_fidelity_probe)
16218            .expect("typed side effect");
16219        let commands = ctx.take_commands().expect("side-effect command");
16220        assert_eq!(
16221            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16222                .expect("recorded side effect"),
16223            value
16224        );
16225
16226        let replay = workflow_context_with_codec(
16227            vec![history_event(
16228                "SideEffectRecorded",
16229                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16230            )],
16231            DEFAULT_CODEC,
16232        );
16233        assert_eq!(
16234            replay
16235                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16236                .expect("replayed typed side effect"),
16237            value
16238        );
16239    }
16240
16241    #[test]
16242    fn ordered_side_effects_share_the_durable_command_stream() {
16243        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16244        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16245        let ctx = workflow_context(vec![
16246            history_event(
16247                "SideEffectRecorded",
16248                json!({"sequence": 1, "result": first}),
16249            ),
16250            history_event(
16251                "SideEffectRecorded",
16252                json!({"sequence": 2, "result": second}),
16253            ),
16254        ]);
16255        let first: String = ctx
16256            .side_effect(|| panic!("first callback must not run"))
16257            .expect("first replay");
16258        let second: i32 = ctx
16259            .side_effect(|| panic!("second callback must not run"))
16260            .expect("second replay");
16261        assert_eq!(first, "first");
16262        assert_eq!(second, 29);
16263        ctx.ensure_history_consumed().expect("ordered history");
16264
16265        let reordered = workflow_context(vec![history_event(
16266            "VersionMarkerRecorded",
16267            json!({
16268                "sequence": 1,
16269                "change_id": "before-side-effect",
16270                "version": 1,
16271                "min_supported": 1,
16272                "max_supported": 1,
16273            }),
16274        )]);
16275        let error = reordered
16276            .side_effect(|| "new".to_string())
16277            .expect_err("command reordering must fail");
16278        assert!(matches!(
16279            error,
16280            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16281                if reason == "recorded_command_mismatch"
16282        ));
16283    }
16284
16285    #[test]
16286    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16287        let ctx = workflow_context(Vec::new());
16288        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16289        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16290        assert!(ctx.patched("new-search").expect("patch"));
16291        ctx.deprecate_patch("new-search").expect("deprecate patch");
16292        let commands = ctx.take_commands().expect("commands");
16293        assert_eq!(commands.len(), 2);
16294        assert_eq!(commands[0]["type"], "record_version_marker");
16295        assert_eq!(commands[0]["version"], 2);
16296        assert_eq!(commands[1]["change_id"], "new-search");
16297
16298        let replay = workflow_context(vec![history_event(
16299            "VersionMarkerRecorded",
16300            json!({
16301                "sequence": 1,
16302                "change_id": "checkout-v2",
16303                "version": 2,
16304                "min_supported": 1,
16305                "max_supported": 2,
16306            }),
16307        )]);
16308        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16309        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16310        assert!(replay.take_commands().expect("commands").is_empty());
16311        replay.ensure_history_consumed().expect("history consumed");
16312    }
16313
16314    #[test]
16315    fn version_markers_reject_incompatible_or_malformed_history() {
16316        let incompatible = workflow_context(vec![history_event(
16317            "VersionMarkerRecorded",
16318            json!({
16319                "sequence": 1,
16320                "change_id": "checkout-v2",
16321                "version": 1,
16322                "min_supported": 1,
16323                "max_supported": 2,
16324            }),
16325        )]);
16326        let error = incompatible
16327            .get_version("checkout-v2", 2, 3)
16328            .expect_err("old version is unsupported");
16329        assert!(matches!(
16330            error,
16331            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16332                if reason == "version_marker_incompatible_range"
16333        ));
16334
16335        for (history, reason) in [
16336            (
16337                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16338                "side_effect_result_missing",
16339            ),
16340            (
16341                vec![history_event(
16342                    "SideEffectRecorded",
16343                    json!({
16344                        "sequence": 1,
16345                        "result": {"codec": "avro", "blob": "not-base64"},
16346                    }),
16347                )],
16348                "side_effect_payload_incompatible",
16349            ),
16350            (
16351                vec![history_event(
16352                    "SideEffectRecorded",
16353                    json!({"sequence": 1, "result": {"unwrapped": true}}),
16354                )],
16355                "side_effect_payload_malformed",
16356            ),
16357            (
16358                vec![history_event(
16359                    "VersionMarkerRecorded",
16360                    json!({
16361                        "sequence": 1,
16362                        "change_id": "change",
16363                        "version": 1,
16364                        "min_supported": 2,
16365                        "max_supported": 1,
16366                    }),
16367                )],
16368                "version_marker_history_range_invalid",
16369            ),
16370        ] {
16371            let error = WorkflowState::new(
16372                history,
16373                "rust-workers".to_string(),
16374                DEFAULT_CODEC.to_string(),
16375                None,
16376            )
16377            .expect_err("malformed history must fail");
16378            assert!(matches!(
16379                error,
16380                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16381                    if actual == reason
16382            ));
16383        }
16384    }
16385
16386    #[test]
16387    fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16388        let history = vec![history_event(
16389            "SearchAttributesUpserted",
16390            json!({
16391                "sequence": 1,
16392                "attributes": {"customer_tier": "gold"},
16393                "attribute_types": {"customer_tier": "keyword"},
16394                "merged": {"customer_tier": "gold"}
16395            }),
16396        )];
16397
16398        let matching = workflow_context(history.clone());
16399        matching
16400            .upsert_search_attributes(
16401                SearchAttributeUpdate::new()
16402                    .keyword("customer_tier", "gold")
16403                    .expect("keyword update"),
16404            )
16405            .expect("matching typed update must replay");
16406        matching
16407            .ensure_history_consumed()
16408            .expect("history consumed");
16409
16410        let changed_type = workflow_context(history.clone());
16411        let error = changed_type
16412            .upsert_search_attributes(
16413                SearchAttributeUpdate::new()
16414                    .string("customer_tier", "gold")
16415                    .expect("string update"),
16416            )
16417            .expect_err("same JSON value with a different declaration must be nondeterministic");
16418        let Error::NonDeterministicReplay(failure) = error else {
16419            panic!("typed identity drift must be a replay failure");
16420        };
16421        assert_eq!(failure.reason, "search_attribute_type_mismatch");
16422        assert_eq!(failure.sequence, Some(1));
16423
16424        let changed_value = workflow_context(history);
16425        let error = changed_value
16426            .upsert_search_attributes(
16427                SearchAttributeUpdate::new()
16428                    .keyword("customer_tier", "platinum")
16429                    .expect("keyword update"),
16430            )
16431            .expect_err("changed values must be nondeterministic");
16432        let Error::NonDeterministicReplay(failure) = error else {
16433            panic!("value drift must be a replay failure");
16434        };
16435        assert_eq!(failure.reason, "search_attribute_value_mismatch");
16436    }
16437
16438    #[test]
16439    fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16440        let history = vec![history_event(
16441            "SearchAttributesUpserted",
16442            json!({
16443                "sequence": 1,
16444                "attributes": {"customer_tier": "gold"},
16445                "merged": {"customer_tier": "gold"}
16446            }),
16447        )];
16448
16449        for update in [
16450            SearchAttributeUpdate::new()
16451                .keyword("customer_tier", "gold")
16452                .expect("keyword update"),
16453            SearchAttributeUpdate::new()
16454                .string("customer_tier", "gold")
16455                .expect("string update"),
16456        ] {
16457            let restarted = workflow_context(history.clone());
16458            restarted
16459                .upsert_search_attributes(update)
16460                .expect("legacy history constrains values but has unknown type identity");
16461            restarted
16462                .ensure_history_consumed()
16463                .expect("history consumed");
16464        }
16465    }
16466
16467    #[test]
16468    fn search_attribute_command_emits_canonical_types() {
16469        let ctx = workflow_context(Vec::new());
16470        ctx.upsert_search_attributes(
16471            SearchAttributeUpdate::new()
16472                .keyword("customer_tier", "gold")
16473                .expect("keyword update")
16474                .int("attempts", 3)
16475                .expect("int update")
16476                .delete("obsolete")
16477                .expect("delete update"),
16478        )
16479        .expect("valid search attributes");
16480
16481        assert_eq!(
16482            ctx.take_commands().expect("commands"),
16483            vec![json!({
16484                "type": "upsert_search_attributes",
16485                "attributes": {
16486                    "attempts": 3,
16487                    "customer_tier": "gold",
16488                    "obsolete": null
16489                },
16490                "attribute_types": {
16491                    "attempts": "int",
16492                    "customer_tier": "keyword"
16493                }
16494            })]
16495        );
16496    }
16497
16498    #[test]
16499    fn duplicate_side_effects_and_version_markers_are_rejected() {
16500        let duplicate_side_effect = WorkflowState::new(
16501            vec![
16502                history_event(
16503                    "SideEffectRecorded",
16504                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16505                ),
16506                history_event(
16507                    "SideEffectRecorded",
16508                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16509                ),
16510            ],
16511            "rust-workers".to_string(),
16512            DEFAULT_CODEC.to_string(),
16513            None,
16514        )
16515        .expect_err("duplicate side effect");
16516        assert!(matches!(
16517            duplicate_side_effect,
16518            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16519                if reason == "duplicate_side_effect_record"
16520        ));
16521
16522        let marker = |sequence| {
16523            history_event(
16524                "VersionMarkerRecorded",
16525                json!({
16526                    "sequence": sequence,
16527                    "change_id": "same-change",
16528                    "version": 1,
16529                    "min_supported": 1,
16530                    "max_supported": 1,
16531                }),
16532            )
16533        };
16534        let duplicate_marker = WorkflowState::new(
16535            vec![marker(1), marker(3)],
16536            "rust-workers".to_string(),
16537            DEFAULT_CODEC.to_string(),
16538            None,
16539        )
16540        .expect_err("duplicate marker");
16541        assert!(matches!(
16542            duplicate_marker,
16543            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16544                if reason == "duplicate_version_marker"
16545        ));
16546    }
16547
16548    #[test]
16549    fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16550        let mut state = WorkflowState::new(
16551            Vec::new(),
16552            "rust-workers".to_string(),
16553            DEFAULT_CODEC.to_string(),
16554            None,
16555        )
16556        .expect("workflow state");
16557        state.workflow_command_identity = "command-7".to_string();
16558        let context = WorkflowContext {
16559            state: Arc::new(Mutex::new(state)),
16560        };
16561        let item =
16562            WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16563
16564        context
16565            .append_workflow_stream("output", &[item], Some(10))
16566            .expect("append command");
16567        context
16568            .error_workflow_stream("output", "producer failed", None)
16569            .expect("error command");
16570        let commands = context.take_commands().expect("commands");
16571
16572        assert_eq!(commands[0]["type"], "record_side_effect");
16573        assert_eq!(
16574            commands[0]["workflow_stream"]["command_identity"],
16575            "command-7"
16576        );
16577        assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16578        assert_eq!(
16579            commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16580            "dw-stream:command-7:0:0"
16581        );
16582        assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16583
16584        let recorded = history_event(
16585            "SideEffectRecorded",
16586            json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16587        );
16588        let mut replay_state = WorkflowState::new(
16589            vec![recorded],
16590            "rust-workers".to_string(),
16591            DEFAULT_CODEC.to_string(),
16592            None,
16593        )
16594        .expect("replay state");
16595        replay_state.workflow_command_identity = "command-7".to_string();
16596        let replay_context = WorkflowContext {
16597            state: Arc::new(Mutex::new(replay_state)),
16598        };
16599        replay_context
16600            .append_workflow_stream(
16601                "output",
16602                &[WorkflowStreamAppendItem::from_reference(
16603                    "s3://bucket/item.avro",
16604                )],
16605                Some(10),
16606            )
16607            .expect("replayed append");
16608        assert!(replay_context
16609            .take_commands()
16610            .expect("replayed commands")
16611            .is_empty());
16612    }
16613
16614    #[test]
16615    fn workflow_stream_authoring_requires_server_durable_command_identity() {
16616        let context = workflow_context(Vec::new());
16617        let error = context
16618            .append_workflow_stream(
16619                "output",
16620                &[WorkflowStreamAppendItem::from_reference(
16621                    "s3://bucket/item.avro",
16622                )],
16623                None,
16624            )
16625            .expect_err("stream append without durable command identity must fail closed");
16626
16627        assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16628        assert!(context.take_commands().expect("commands").is_empty());
16629    }
16630
16631    #[test]
16632    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16633        fn worker(calls: Arc<AtomicUsize>) -> Worker {
16634            let client = Client::new("http://127.0.0.1:8080").expect("client");
16635            let mut worker = Worker::new(client, "rust-workers");
16636            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16637                let calls = Arc::clone(&calls);
16638                async move {
16639                    let captured = ctx.side_effect(|| {
16640                        calls.fetch_add(1, Ordering::SeqCst);
16641                        "captured-once".to_string()
16642                    })?;
16643                    let version = ctx.get_version("cold-restart", 1, 2)?;
16644                    Ok(json!({"captured": captured, "version": version}))
16645                }
16646            });
16647            worker
16648        }
16649
16650        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16651            WorkflowTask {
16652                task_id: "wft-side-effect-version".to_string(),
16653                workflow_command_id: None,
16654                workflow_id: Some("wf-side-effect-version".to_string()),
16655                run_id: Some("run-side-effect-version".to_string()),
16656                workflow_type: "rust.side-effect-version".to_string(),
16657                cancel_requested: false,
16658                payload_codec: DEFAULT_CODEC.to_string(),
16659                arguments: Some(
16660                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16661                ),
16662                history_events,
16663                total_history_events: None,
16664                history_size_bytes: None,
16665                continue_as_new_recommended: None,
16666                history_budget_pressure: None,
16667                next_history_page_token: None,
16668                workflow_task_attempt: 1,
16669                workflow_signal_id: None,
16670                signal_name: None,
16671                signal_arguments: None,
16672                workflow_update_id: None,
16673                update_name: None,
16674                lease_owner: Some("rust-worker".to_string()),
16675            }
16676        }
16677
16678        let calls = Arc::new(AtomicUsize::new(0));
16679        let initial = worker(Arc::clone(&calls))
16680            .execute_workflow_task(task(Vec::new()))
16681            .expect("initial execution");
16682        assert_eq!(
16683            initial
16684                .iter()
16685                .map(|command| &command["type"])
16686                .collect::<Vec<_>>(),
16687            vec![
16688                "record_side_effect",
16689                "record_version_marker",
16690                "complete_workflow"
16691            ]
16692        );
16693        assert_eq!(calls.load(Ordering::SeqCst), 1);
16694
16695        let restarted = worker(Arc::clone(&calls));
16696        let replayed = restarted
16697            .execute_workflow_task(task(vec![
16698                history_event(
16699                    "SideEffectRecorded",
16700                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16701                ),
16702                history_event(
16703                    "VersionMarkerRecorded",
16704                    json!({
16705                        "sequence": 2,
16706                        "change_id": "cold-restart",
16707                        "version": 2,
16708                        "min_supported": 1,
16709                        "max_supported": 2,
16710                    }),
16711                ),
16712            ]))
16713            .expect("cold replay");
16714        assert_eq!(replayed.len(), 1);
16715        assert_eq!(replayed[0]["type"], "complete_workflow");
16716        assert_eq!(calls.load(Ordering::SeqCst), 1);
16717    }
16718
16719    #[test]
16720    fn side_effect_replay_rejects_changed_rust_value_type() {
16721        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16722        let ctx = workflow_context(vec![history_event(
16723            "SideEffectRecorded",
16724            json!({"sequence": 1, "result": result}),
16725        )]);
16726        let error = ctx
16727            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16728            .expect_err("changed type must fail replay");
16729        assert!(matches!(
16730            error,
16731            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16732                if reason == "side_effect_type_mismatch"
16733        ));
16734    }
16735
16736    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16737        vec![
16738            history_event(
16739                "ActivityScheduled",
16740                json!({
16741                    "sequence": 1,
16742                    "activity_type": "flaky",
16743                    "activity_execution_id": "act-1",
16744                    "activity": {
16745                        "id": "act-1",
16746                        "sequence": 1,
16747                        "type": "flaky",
16748                        "queue": "critical-activities",
16749                        "execution_mode": null,
16750                        "retry_policy": {
16751                            "snapshot_version": 1,
16752                            "max_attempts": 3,
16753                            "backoff_seconds": [2, 4],
16754                            "start_to_close_timeout": 30,
16755                            "schedule_to_start_timeout": 5,
16756                            "schedule_to_close_timeout": 90,
16757                            "heartbeat_timeout": 10,
16758                            "non_retryable_error_types": ["PermanentError"]
16759                        }
16760                    }
16761                }),
16762            ),
16763            history_event(
16764                "ActivityStarted",
16765                json!({
16766                    "sequence": 1,
16767                    "activity_type": "flaky",
16768                    "activity_execution_id": "act-1",
16769                    "activity_attempt_id": "attempt-1",
16770                    "attempt_number": 1
16771                }),
16772            ),
16773            history_event(
16774                "ActivityRetryScheduled",
16775                json!({
16776                    "sequence": 1,
16777                    "activity_type": "flaky",
16778                    "activity_execution_id": "act-1",
16779                    "activity_attempt_id": "attempt-1",
16780                    "attempt_number": 1,
16781                    "retry_after_attempt": 1,
16782                    "retry_backoff_seconds": 2,
16783                    "failure_category": "activity",
16784                    "exception_type": "TransientError"
16785                }),
16786            ),
16787            history_event(
16788                "ActivityStarted",
16789                json!({
16790                    "sequence": 1,
16791                    "activity_type": "flaky",
16792                    "activity_execution_id": "act-1",
16793                    "activity_attempt_id": "attempt-2",
16794                    "attempt_number": 2
16795                }),
16796            ),
16797            history_event(
16798                "ActivityCompleted",
16799                json!({
16800                    "sequence": 1,
16801                    "activity_type": "flaky",
16802                    "activity_execution_id": "act-1",
16803                    "activity_attempt_id": "attempt-2",
16804                    "attempt_number": 2,
16805                    "payload_codec": DEFAULT_CODEC,
16806                    "result": fixture_envelope(json!({"status":"recovered"}))
16807                }),
16808            ),
16809        ]
16810    }
16811
16812    fn retry_activity_options() -> ActivityOptions {
16813        ActivityOptions::new()
16814            .task_queue("critical-activities")
16815            .retry_policy(
16816                ActivityRetryPolicy::new(3)
16817                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
16818                    .non_retryable_error_type("PermanentError"),
16819            )
16820            .start_to_close_timeout(Duration::from_secs(30))
16821            .schedule_to_start_timeout(Duration::from_secs(5))
16822            .schedule_to_close_timeout(Duration::from_secs(90))
16823            .heartbeat_timeout(Duration::from_secs(10))
16824    }
16825
16826    #[test]
16827    fn fixed_avro_value_round_trips_json_values() {
16828        let value = json!({"greeting": "hello", "count": 3, "ok": true});
16829        let envelope = PayloadEnvelope::avro(&value).expect("encode");
16830        assert_eq!(envelope.codec, DEFAULT_CODEC);
16831        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
16832    }
16833
16834    #[tokio::test]
16835    async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
16836        let client = Client::new("http://127.0.0.1:8080").expect("client");
16837        let mut worker = Worker::new(client, "rust-workers");
16838        worker.register_typed_workflow(
16839            "typed.contract.workflow",
16840            |_ctx, input: TypedContract| async move { Ok(input) },
16841        );
16842        worker.register_typed_activity(
16843            "typed.contract.activity",
16844            |_ctx, input: TypedContract| async move { Ok(input) },
16845        );
16846
16847        let expected = typed_contract();
16848        let arguments = AvroValue::Array(vec![
16849            AvroValue::from_serialize(&expected).expect("typed request")
16850        ]);
16851        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
16852        let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
16853        workflow.arguments = Some(envelope.clone());
16854        let commands = worker
16855            .execute_workflow_task(workflow)
16856            .expect("typed workflow task");
16857        let workflow_result: TypedContract =
16858            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16859                .expect("workflow result envelope")
16860                .deserialize()
16861                .expect("workflow result type");
16862        assert_eq!(workflow_result, expected);
16863
16864        let activity = ActivityTask {
16865            task_id: "typed-contract-activity".to_string(),
16866            activity_attempt_id: Some("typed-contract-attempt".to_string()),
16867            attempt_id: None,
16868            activity_type: "typed.contract.activity".to_string(),
16869            payload_codec: DEFAULT_CODEC.to_string(),
16870            arguments: Some(envelope),
16871            attempt_number: 1,
16872            lease_owner: Some("rust-worker".to_string()),
16873        };
16874        let activity_result: TypedContract = worker
16875            .execute_activity_task(activity)
16876            .await
16877            .expect("typed activity task")
16878            .deserialize()
16879            .expect("activity result type");
16880        assert_eq!(activity_result, expected);
16881    }
16882
16883    #[tokio::test]
16884    async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
16885        let client = Client::new("http://127.0.0.1:8080").expect("client");
16886        let mut worker = Worker::new(client, "rust-workers");
16887        worker.register_typed_workflow(
16888            "typed.shape.workflow",
16889            |_ctx, input: TypedContract| async move { Ok(input) },
16890        );
16891        worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
16892            Ok(f64::NAN)
16893        });
16894
16895        let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
16896        workflow.arguments = Some(
16897            encode_typed_envelope(
16898                &AvroValue::Array(vec![
16899                    AvroValue::String("first".to_string()),
16900                    AvroValue::String("second".to_string()),
16901                ]),
16902                DEFAULT_CODEC,
16903            )
16904            .expect("malformed typed arguments"),
16905        );
16906        let commands = worker
16907            .execute_workflow_task(workflow)
16908            .expect("shape mismatch becomes a workflow failure");
16909        let message = commands[0]["message"].as_str().expect("failure message");
16910        assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
16911        assert!(message.contains(type_name::<TypedContract>()));
16912        assert!(message.contains("task carried 2 arguments"));
16913
16914        let activity = ActivityTask {
16915            task_id: "typed-unsupported-activity".to_string(),
16916            activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
16917            attempt_id: None,
16918            activity_type: "typed.unsupported.activity".to_string(),
16919            payload_codec: DEFAULT_CODEC.to_string(),
16920            arguments: Some(
16921                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
16922                    .expect("unit arguments"),
16923            ),
16924            attempt_number: 1,
16925            lease_owner: Some("rust-worker".to_string()),
16926        };
16927        let Error::HandlerType {
16928            handler_kind,
16929            handler_name,
16930            value_kind,
16931            rust_type,
16932            message,
16933        } = worker
16934            .execute_activity_task(activity)
16935            .await
16936            .expect_err("non-finite handler output must fail")
16937        else {
16938            panic!("expected contextual handler type failure");
16939        };
16940        assert_eq!(handler_kind, HandlerKind::Activity);
16941        assert_eq!(handler_name, "typed.unsupported.activity");
16942        assert_eq!(value_kind, HandlerValueKind::Result);
16943        assert_eq!(rust_type, type_name::<f64>());
16944        assert!(message.contains("non_finite_float"));
16945    }
16946
16947    #[tokio::test]
16948    async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
16949        #[derive(Clone, Default)]
16950        struct State {
16951            observed: Option<TypedContract>,
16952        }
16953
16954        let client = Client::new("http://127.0.0.1:8080").expect("client");
16955        let mut worker = Worker::new(client, "rust-workers");
16956        worker.register_typed_replayed_workflow(
16957            "typed.contract.replayed",
16958            State::default,
16959            |ctx, input: TypedContract, state| async move {
16960                let result: TypedContract =
16961                    ctx.activity_typed("typed.contract.activity", input).await?;
16962                state.update(|current| current.observed = Some(result.clone()))?;
16963                Ok(result)
16964            },
16965        );
16966        worker.register_replayed_query::<State, _, _>(
16967            "typed.contract.replayed",
16968            "observed",
16969            |_ctx, state, _args| async move {
16970                Ok(json!(state.observed.as_ref().map(|value| value.signed)))
16971            },
16972        );
16973
16974        let expected = typed_contract();
16975        let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
16976        let workflow_arguments =
16977            encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
16978                .expect("workflow arguments");
16979        let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
16980        let task = QueryTask {
16981            query_task_id: "typed-replay-query".to_string(),
16982            query_task_attempt: 1,
16983            lease_owner: Some("rust-worker".to_string()),
16984            workflow_id: Some("typed-replay".to_string()),
16985            run_id: Some("typed-replay-run".to_string()),
16986            workflow_type: "typed.contract.replayed".to_string(),
16987            query_name: "observed".to_string(),
16988            payload_codec: DEFAULT_CODEC.to_string(),
16989            workflow_arguments: Some(workflow_arguments),
16990            query_arguments: Some(
16991                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
16992                    .expect("query arguments"),
16993            ),
16994            history_events: vec![
16995                history_event(
16996                    "ActivityScheduled",
16997                    json!({
16998                        "sequence": 1,
16999                        "activity_type": "typed.contract.activity"
17000                    }),
17001                ),
17002                history_event(
17003                    "ActivityCompleted",
17004                    json!({
17005                        "sequence": 1,
17006                        "activity_type": "typed.contract.activity",
17007                        "payload_codec": DEFAULT_CODEC,
17008                        "result": result
17009                    }),
17010                ),
17011            ],
17012            history_export: None,
17013            run_status: Some("completed".to_string()),
17014        };
17015
17016        assert_eq!(
17017            worker
17018                .execute_query_task(task)
17019                .await
17020                .expect("typed replay query")
17021                .deserialize::<i64>()
17022                .expect("query result"),
17023            expected.signed
17024        );
17025    }
17026
17027    #[tokio::test]
17028    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17029        let client = Client::new("http://127.0.0.1:8080").expect("client");
17030        let mut worker = Worker::new(client, "rust-workers");
17031        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17032        worker
17033            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17034        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17035            Ok(input)
17036        });
17037        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17038            Ok(input)
17039        });
17040        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17041            Ok(AvroValue::Array(
17042                ctx.wait_signal_avro_value("changed").await?,
17043            ))
17044        });
17045
17046        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17047        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17048
17049        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17050        workflow.arguments = Some(envelope.clone());
17051        let commands = worker
17052            .execute_workflow_task(workflow)
17053            .expect("typed workflow task");
17054        assert_eq!(commands[0]["type"], "complete_workflow");
17055        assert_eq!(
17056            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17057                .expect("typed workflow result"),
17058            arguments
17059        );
17060
17061        let activity = ActivityTask {
17062            task_id: "activity-typed".to_string(),
17063            activity_attempt_id: Some("attempt-typed".to_string()),
17064            attempt_id: None,
17065            activity_type: "typed.activity".to_string(),
17066            payload_codec: DEFAULT_CODEC.to_string(),
17067            arguments: Some(envelope.clone()),
17068            attempt_number: 1,
17069            lease_owner: Some("rust-worker".to_string()),
17070        };
17071        assert_eq!(
17072            worker
17073                .execute_activity_task(activity)
17074                .await
17075                .expect("typed activity result"),
17076            arguments
17077        );
17078
17079        let query = QueryTask {
17080            query_task_id: "query-typed".to_string(),
17081            query_task_attempt: 1,
17082            lease_owner: Some("rust-worker".to_string()),
17083            workflow_id: Some("typed-1".to_string()),
17084            run_id: Some("run-typed".to_string()),
17085            workflow_type: "typed.echo".to_string(),
17086            query_name: "inspect".to_string(),
17087            payload_codec: DEFAULT_CODEC.to_string(),
17088            workflow_arguments: Some(
17089                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17090                    .expect("workflow input"),
17091            ),
17092            query_arguments: Some(envelope.clone()),
17093            history_events: Vec::new(),
17094            history_export: None,
17095            run_status: Some("running".to_string()),
17096        };
17097        assert_eq!(
17098            worker
17099                .execute_query_task(query)
17100                .await
17101                .expect("typed query result"),
17102            arguments
17103        );
17104
17105        let mut update = workflow_task(
17106            "typed.echo",
17107            vec![history_event(
17108                "UpdateAccepted",
17109                json!({
17110                    "update_id": "update-typed",
17111                    "update_name": "replace",
17112                    "arguments": envelope.clone(),
17113                }),
17114            )],
17115            DEFAULT_CODEC,
17116        );
17117        update.workflow_update_id = Some("update-typed".to_string());
17118        update.update_name = Some("replace".to_string());
17119        let commands = worker
17120            .execute_workflow_task(update)
17121            .expect("typed update task");
17122        assert_eq!(commands[0]["type"], "complete_update");
17123        assert_eq!(
17124            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17125                .expect("typed update result"),
17126            arguments
17127        );
17128
17129        let mut signal = workflow_task(
17130            "typed.signal",
17131            vec![history_event(
17132                "SignalReceived",
17133                json!({
17134                    "signal_id": "signal-typed",
17135                    "signal_name": "changed",
17136                    "arguments": envelope.clone(),
17137                }),
17138            )],
17139            DEFAULT_CODEC,
17140        );
17141        signal.workflow_signal_id = Some("signal-typed".to_string());
17142        signal.signal_name = Some("changed".to_string());
17143        signal.signal_arguments = Some(envelope);
17144        let commands = worker
17145            .execute_workflow_task(signal)
17146            .expect("typed signal resume");
17147        assert_eq!(
17148            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17149                .expect("typed signal result"),
17150            arguments
17151        );
17152    }
17153
17154    #[tokio::test]
17155    async fn typed_helpers_never_parse_json_inspection_projection() {
17156        let collision_values = projection_collision_probe();
17157        let expected = AvroValue::Array(collision_values.clone());
17158        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17159
17160        let activity_context = workflow_context_with_codec(
17161            vec![history_event(
17162                "ActivityCompleted",
17163                json!({
17164                    "sequence": 1,
17165                    "activity_type": "collision.activity",
17166                    "payload_codec": DEFAULT_CODEC,
17167                    "result": envelope.clone(),
17168                }),
17169            )],
17170            DEFAULT_CODEC,
17171        );
17172        assert_eq!(
17173            activity_context
17174                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17175                .await
17176                .expect("typed activity collision result"),
17177            expected
17178        );
17179
17180        let signal_context = workflow_context_with_codec(
17181            vec![
17182                history_event(
17183                    "SignalWaitOpened",
17184                    json!({"sequence": 1, "signal_name": "collision"}),
17185                ),
17186                history_event(
17187                    "SignalApplied",
17188                    json!({
17189                        "sequence": 1,
17190                        "signal_name": "collision",
17191                        "payload_codec": DEFAULT_CODEC,
17192                        "value": envelope.clone(),
17193                    }),
17194                ),
17195            ],
17196            DEFAULT_CODEC,
17197        );
17198        assert_eq!(
17199            signal_context
17200                .wait_signal_avro_value("collision")
17201                .await
17202                .expect("typed signal collision arguments"),
17203            collision_values
17204        );
17205
17206        let child_context = workflow_context_with_codec(
17207            vec![
17208                history_event(
17209                    "ChildWorkflowScheduled",
17210                    json!({
17211                        "sequence": 1,
17212                        "child_workflow_instance_id": "collision-child",
17213                        "child_workflow_run_id": "collision-run",
17214                        "child_workflow_type": "collision.child",
17215                    }),
17216                ),
17217                history_event(
17218                    "ChildRunCompleted",
17219                    json!({
17220                        "sequence": 1,
17221                        "child_workflow_instance_id": "collision-child",
17222                        "child_workflow_run_id": "collision-run",
17223                        "child_workflow_type": "collision.child",
17224                        "payload_codec": DEFAULT_CODEC,
17225                        "result": envelope,
17226                    }),
17227                ),
17228            ],
17229            DEFAULT_CODEC,
17230        );
17231        let child = child_context
17232            .start_child_workflow_avro_value(
17233                "collision.child",
17234                ChildWorkflowOptions::new("collision-workers"),
17235                AvroValue::Array(Vec::new()),
17236            )
17237            .await
17238            .expect("typed child collision result");
17239        assert_eq!(child.result, expected);
17240    }
17241
17242    #[tokio::test]
17243    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17244        let client = Client::new("http://127.0.0.1:8080").expect("client");
17245        let mut worker = Worker::new(client, "rust-workers");
17246        worker.register_replayed_workflow_avro_value(
17247            "typed.replayed",
17248            || (),
17249            |_ctx, input, _state| async move { Ok(input) },
17250        );
17251        worker.register_replayed_query_avro_value::<(), _, _>(
17252            "typed.replayed",
17253            "inspect",
17254            |ctx, _state, args| async move {
17255                let mut signals = ctx.signals_avro_value("collision");
17256                let signal = signals
17257                    .pop()
17258                    .map(AvroValue::Array)
17259                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17260                Ok(AvroValue::Array(vec![
17261                    ctx.workflow_input_avro_value().clone(),
17262                    signal,
17263                    args,
17264                ]))
17265            },
17266        );
17267        let arguments = AvroValue::Array(projection_collision_probe());
17268        let signal_arguments =
17269            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17270        let task = QueryTask {
17271            query_task_id: "query-typed-replay".to_string(),
17272            query_task_attempt: 1,
17273            lease_owner: Some("rust-worker".to_string()),
17274            workflow_id: Some("typed-replay".to_string()),
17275            run_id: Some("run-typed-replay".to_string()),
17276            workflow_type: "typed.replayed".to_string(),
17277            query_name: "inspect".to_string(),
17278            payload_codec: DEFAULT_CODEC.to_string(),
17279            workflow_arguments: Some(
17280                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17281            ),
17282            query_arguments: Some(
17283                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17284            ),
17285            history_events: vec![history_event(
17286                "SignalReceived",
17287                json!({
17288                    "signal_id": "collision-signal",
17289                    "signal_name": "collision",
17290                    "workflow_sequence": 1,
17291                    "payload_codec": DEFAULT_CODEC,
17292                    "arguments": signal_arguments,
17293                }),
17294            )],
17295            history_export: None,
17296            run_status: Some("completed".to_string()),
17297        };
17298
17299        assert_eq!(
17300            worker
17301                .execute_query_task(task)
17302                .await
17303                .expect("typed replay query"),
17304            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17305        );
17306    }
17307
17308    #[test]
17309    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17310        let value = BTreeMap::from([(1_i32, "integer key")]);
17311        let error = PayloadEnvelope::avro(&value)
17312            .expect_err("integer map keys must fail")
17313            .to_string();
17314
17315        assert!(error.contains("invalid_map_key"));
17316    }
17317
17318    #[test]
17319    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17320        let envelope = PayloadEnvelope {
17321            codec: "json".to_string(),
17322            blob: r#"{"greeting":"hello"}"#.to_string(),
17323        };
17324
17325        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17326        let diagnostic = error.to_string();
17327        assert!(diagnostic.contains("unsupported_payload_codec"));
17328        assert!(diagnostic.contains("codec=\"avro\""));
17329        assert!(diagnostic.contains("HTTP document transport"));
17330    }
17331
17332    #[test]
17333    fn untagged_json_payload_value_fails_closed() {
17334        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17335            .expect_err("untagged JSON payload values must fail");
17336        let diagnostic = error.to_string();
17337        assert!(diagnostic.contains("unsupported_payload_codec"));
17338        assert!(diagnostic.contains("untagged durable payload"));
17339        assert!(diagnostic.contains("HTTP document transport"));
17340    }
17341
17342    #[test]
17343    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17344        let envelope = PayloadEnvelope {
17345            codec: DEFAULT_CODEC.to_string(),
17346            blob: BASE64.encode([0x01]),
17347        };
17348
17349        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17350        assert!(error.to_string().contains("invalid_payload_framing"));
17351    }
17352
17353    #[tokio::test]
17354    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17355        let server = MockWorkerServer::start();
17356        let client = Client::builder(server.base_url())
17357            .timeout(Duration::from_secs(2))
17358            .build()
17359            .expect("client");
17360        let invalid_commands = [
17361            json!({
17362                "type": "complete_workflow",
17363                "result": {"codec": "json", "blob": null}
17364            }),
17365            json!({
17366                "type": "schedule_activity",
17367                "arguments": {"codec": "yaml", "blob": "ignored"}
17368            }),
17369            json!({
17370                "type": "start_child_workflow",
17371                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17372            }),
17373            json!({"type": "continue_as_new", "arguments": []}),
17374            json!({"type": "complete_update"}),
17375            json!({"type": "record_side_effect", "result": null}),
17376            json!({
17377                "type": "start_service_operation",
17378                "payload_codec": DEFAULT_CODEC,
17379                "request_payload": "raw-avro-bytes"
17380            }),
17381        ];
17382
17383        for command in invalid_commands {
17384            let error = client
17385                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17386                .await
17387                .expect_err("invalid durable payload must fail locally");
17388            let diagnostic = error.to_string();
17389            assert!(
17390                diagnostic.contains("unsupported_payload_codec")
17391                    || diagnostic.contains("invalid_payload_envelope")
17392                    || diagnostic.contains("untagged durable payload"),
17393                "unexpected validation diagnostic: {diagnostic}"
17394            );
17395        }
17396
17397        assert_eq!(
17398            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17399            0,
17400            "invalid command payloads must not reach HTTP transport"
17401        );
17402    }
17403
17404    #[test]
17405    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17406        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17407        let commands = [
17408            json!({"type": "complete_workflow", "result": envelope.clone()}),
17409            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17410            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17411            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17412            json!({"type": "complete_update", "result": envelope.clone()}),
17413            json!({"type": "record_side_effect", "result": envelope.clone()}),
17414            json!({
17415                "type": "start_service_operation",
17416                "payload_codec": DEFAULT_CODEC,
17417                "request_payload": envelope.clone()
17418            }),
17419            json!({
17420                "type": "complete_workflow",
17421                "result": envelope,
17422                "metadata": {
17423                    "codec": "json",
17424                    "payload_codec": "customer-codec",
17425                    "result": {"codec": "yaml", "blob": null}
17426                }
17427            }),
17428        ];
17429
17430        validate_workflow_task_commands(&commands)
17431            .expect("customer metadata must not become a protocol codec declaration");
17432    }
17433
17434    #[test]
17435    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17436        assert_eq!(
17437            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17438            AvroValue::Array(Vec::new())
17439        );
17440        assert_eq!(
17441            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17442            AvroValue::Array(Vec::new())
17443        );
17444
17445        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17446        signal.signal_name = Some("empty-signal".to_string());
17447        signal.signal_arguments = None;
17448        let decoded = decode_resume_signal(&signal)
17449            .expect("valid Avro signal")
17450            .expect("named signal resumes the workflow");
17451        assert!(decoded.arguments.is_empty());
17452    }
17453
17454    #[tokio::test]
17455    async fn malformed_task_level_codecs_become_pre_handler_failures() {
17456        let client = Client::new("http://127.0.0.1:8080").expect("client");
17457        let mut worker = Worker::new(client, "rust-workers");
17458        let handler_calls = Arc::new(AtomicUsize::new(0));
17459
17460        let calls = Arc::clone(&handler_calls);
17461        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17462            calls.fetch_add(1, Ordering::SeqCst);
17463            async move { Ok(Value::Null) }
17464        });
17465        let calls = Arc::clone(&handler_calls);
17466        worker.register_activity("codec.activity", move |_ctx, _args| {
17467            calls.fetch_add(1, Ordering::SeqCst);
17468            async move { Ok(Value::Null) }
17469        });
17470        let calls = Arc::clone(&handler_calls);
17471        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17472            calls.fetch_add(1, Ordering::SeqCst);
17473            async move { Ok(Value::Null) }
17474        });
17475
17476        let mut failures = Vec::new();
17477        for codec_case in [
17478            InvalidTaskPayloadCodec::Missing,
17479            InvalidTaskPayloadCodec::Null,
17480            InvalidTaskPayloadCodec::NonString,
17481        ] {
17482            let mut workflow = json!({
17483                "task_id": format!("workflow-{}", codec_case.label()),
17484                "workflow_type": "codec.workflow"
17485            });
17486            codec_case.apply(&mut workflow);
17487            match serde_json::from_value::<WorkflowTask>(workflow) {
17488                Ok(task) => match worker.execute_workflow_task(task) {
17489                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17490                    outcome => failures.push(format!(
17491                        "workflow {} codec returned {outcome:?}",
17492                        codec_case.label()
17493                    )),
17494                },
17495                Err(error) => failures.push(format!(
17496                    "workflow {} codec failed transport deserialization: {error}",
17497                    codec_case.label()
17498                )),
17499            }
17500
17501            let mut activity = json!({
17502                "task_id": format!("activity-{}", codec_case.label()),
17503                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17504                "activity_type": "codec.activity",
17505                "attempt_number": 1
17506            });
17507            codec_case.apply(&mut activity);
17508            match serde_json::from_value::<ActivityTask>(activity) {
17509                Ok(task) => match worker.execute_activity_task(task).await {
17510                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17511                    outcome => failures.push(format!(
17512                        "activity {} codec returned {outcome:?}",
17513                        codec_case.label()
17514                    )),
17515                },
17516                Err(error) => failures.push(format!(
17517                    "activity {} codec failed transport deserialization: {error}",
17518                    codec_case.label()
17519                )),
17520            }
17521
17522            let mut query = json!({
17523                "query_task_id": format!("query-{}", codec_case.label()),
17524                "workflow_type": "codec.workflow",
17525                "query_name": "known"
17526            });
17527            codec_case.apply(&mut query);
17528            match serde_json::from_value::<QueryTask>(query) {
17529                Ok(task) => match worker.execute_query_task(task).await {
17530                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17531                    outcome => failures.push(format!(
17532                        "query {} codec returned {outcome:?}",
17533                        codec_case.label()
17534                    )),
17535                },
17536                Err(error) => failures.push(format!(
17537                    "query {} codec failed transport deserialization: {error}",
17538                    codec_case.label()
17539                )),
17540            }
17541        }
17542
17543        assert!(failures.is_empty(), "{}", failures.join("\n"));
17544        assert_eq!(
17545            handler_calls.load(Ordering::SeqCst),
17546            0,
17547            "invalid task codecs must not invoke a handler"
17548        );
17549    }
17550
17551    #[tokio::test]
17552    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17553        for codec_case in [
17554            InvalidTaskPayloadCodec::Missing,
17555            InvalidTaskPayloadCodec::Null,
17556            InvalidTaskPayloadCodec::NonString,
17557        ] {
17558            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17559            let client = Client::builder(server.base_url())
17560                .timeout(Duration::from_secs(2))
17561                .build()
17562                .expect("client");
17563            let mut worker = Worker::new(client, "rust-workers")
17564                .worker_id("codec-worker")
17565                .poll_timeout(Duration::from_millis(10));
17566            let handler_calls = Arc::new(AtomicUsize::new(0));
17567
17568            let calls = Arc::clone(&handler_calls);
17569            worker.register_workflow("codec.workflow", move |_ctx, _args| {
17570                calls.fetch_add(1, Ordering::SeqCst);
17571                async move { Ok(Value::Null) }
17572            });
17573            let calls = Arc::clone(&handler_calls);
17574            worker.register_activity("codec.activity", move |_ctx, _args| {
17575                calls.fetch_add(1, Ordering::SeqCst);
17576                async move { Ok(Value::Null) }
17577            });
17578            let calls = Arc::clone(&handler_calls);
17579            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17580                calls.fetch_add(1, Ordering::SeqCst);
17581                async move { Ok(Value::Null) }
17582            });
17583
17584            assert_eq!(
17585                worker.run_once().await.expect("invalid tasks are settled"),
17586                3,
17587                "all {} codec tasks must be handled",
17588                codec_case.label()
17589            );
17590            assert_eq!(
17591                handler_calls.load(Ordering::SeqCst),
17592                0,
17593                "{} task codecs must fail before every handler",
17594                codec_case.label()
17595            );
17596
17597            for path in [
17598                "/api/worker/workflow-tasks/codec-workflow/fail",
17599                "/api/worker/activity-tasks/codec-activity/fail",
17600                "/api/worker/query-tasks/codec-query/fail",
17601            ] {
17602                let body = server.request_body(path);
17603                assert!(
17604                    body["failure"]["message"]
17605                        .as_str()
17606                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
17607                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
17608                    codec_case.label()
17609                );
17610            }
17611            assert_eq!(
17612                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17613                    ["reason"],
17614                "query_payload_decode_failed"
17615            );
17616            for path in [
17617                "/api/worker/workflow-tasks/codec-workflow/complete",
17618                "/api/worker/activity-tasks/codec-activity/complete",
17619                "/api/worker/query-tasks/codec-query/complete",
17620            ] {
17621                assert_eq!(
17622                    server.request_count(path),
17623                    0,
17624                    "invalid {} codec task reached {path}",
17625                    codec_case.label()
17626                );
17627            }
17628        }
17629    }
17630
17631    #[tokio::test]
17632    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17633        let client = Client::new("http://127.0.0.1:8080").expect("client");
17634        let mut worker = Worker::new(client, "rust-workers");
17635        let handler_calls = Arc::new(AtomicUsize::new(0));
17636
17637        let calls = Arc::clone(&handler_calls);
17638        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17639            calls.fetch_add(1, Ordering::SeqCst);
17640            async move { Ok(Value::Null) }
17641        });
17642        let calls = Arc::clone(&handler_calls);
17643        worker.register_activity("codec.activity", move |_ctx, _args| {
17644            calls.fetch_add(1, Ordering::SeqCst);
17645            async move { Ok(Value::Null) }
17646        });
17647        let calls = Arc::clone(&handler_calls);
17648        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17649            calls.fetch_add(1, Ordering::SeqCst);
17650            async move { Ok(Value::Null) }
17651        });
17652        let calls = Arc::clone(&handler_calls);
17653        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17654            calls.fetch_add(1, Ordering::SeqCst);
17655            async move { Ok(Value::Null) }
17656        });
17657
17658        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17659        workflow.payload_codec = "json".to_string();
17660        workflow.arguments = None;
17661        let error = worker
17662            .execute_workflow_task(workflow)
17663            .expect_err("task codec must be checked before workflow invocation");
17664        assert!(error.to_string().contains("unsupported_payload_codec"));
17665
17666        let activity = ActivityTask {
17667            task_id: "activity-invalid-codec".to_string(),
17668            activity_attempt_id: None,
17669            attempt_id: None,
17670            activity_type: "codec.activity".to_string(),
17671            payload_codec: "unknown".to_string(),
17672            arguments: None,
17673            attempt_number: 1,
17674            lease_owner: None,
17675        };
17676        let error = worker
17677            .execute_activity_task(activity)
17678            .await
17679            .expect_err("task codec must be checked before activity invocation");
17680        assert!(error.to_string().contains("unsupported_payload_codec"));
17681
17682        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17683        update.workflow_update_id = Some("update-invalid-codec".to_string());
17684        update.update_name = Some("known".to_string());
17685        update.history_events.push(history_event(
17686            "UpdateAccepted",
17687            json!({
17688                "update_id": "update-invalid-codec",
17689                "update_name": "known",
17690                "arguments": {"codec": "json", "blob": null}
17691            }),
17692        ));
17693        let error = worker
17694            .execute_workflow_task(update)
17695            .expect_err("nested update codec must be checked before handler lookup");
17696        assert!(error.to_string().contains("unsupported_payload_codec"));
17697
17698        let query: QueryTask = serde_json::from_value(json!({
17699            "query_task_id": "query-invalid-codec",
17700            "workflow_type": "codec.workflow",
17701            "query_name": "known",
17702            "payload_codec": DEFAULT_CODEC,
17703            "workflow_arguments": null,
17704            "query_arguments": null,
17705            "history_export": {
17706                "payloads": {"codec": DEFAULT_CODEC},
17707                "signals": [{
17708                    "name": "empty",
17709                    "payload_codec": "json",
17710                    "arguments": null
17711                }]
17712            }
17713        }))
17714        .expect("query task");
17715        let failure = worker
17716            .execute_query_task(query)
17717            .await
17718            .expect_err("exported signal codec must be checked before query invocation");
17719        assert_eq!(failure.reason, "query_payload_decode_failed");
17720        assert!(failure.message.contains("unsupported_payload_codec"));
17721
17722        let exported_history: QueryTask = serde_json::from_value(json!({
17723            "query_task_id": "query-invalid-history-codec",
17724            "workflow_type": "codec.workflow",
17725            "query_name": "known",
17726            "payload_codec": DEFAULT_CODEC,
17727            "history_export": {
17728                "payloads": {"codec": DEFAULT_CODEC},
17729                "history_events": [{
17730                    "type": "ActivityCompleted",
17731                    "payload": {"payload_codec": "unknown", "result": null}
17732                }]
17733            }
17734        }))
17735        .expect("query task");
17736        let failure = worker
17737            .execute_query_task(exported_history)
17738            .await
17739            .expect_err("exported history codec must be checked before query invocation");
17740        assert_eq!(failure.reason, "query_payload_decode_failed");
17741        assert!(failure.message.contains("unsupported_payload_codec"));
17742        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
17743
17744        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17745        unknown_workflow.arguments = None;
17746        unknown_workflow.history_events.push(history_event(
17747            "SignalReceived",
17748            json!({
17749                "signal_name": "empty",
17750                "payload_codec": "json",
17751                "arguments": null
17752            }),
17753        ));
17754        let error = worker
17755            .execute_workflow_task(unknown_workflow)
17756            .expect_err("history codec must precede unknown workflow outcome");
17757        assert!(error.to_string().contains("unsupported_payload_codec"));
17758
17759        let unknown_activity = ActivityTask {
17760            task_id: "activity-unknown".to_string(),
17761            activity_attempt_id: None,
17762            attempt_id: None,
17763            activity_type: "missing".to_string(),
17764            payload_codec: "json".to_string(),
17765            arguments: None,
17766            attempt_number: 1,
17767            lease_owner: None,
17768        };
17769        let error = worker
17770            .execute_activity_task(unknown_activity)
17771            .await
17772            .expect_err("codec must precede unknown activity outcome");
17773        assert!(error.to_string().contains("unsupported_payload_codec"));
17774
17775        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17776        unknown_update.payload_codec = "json".to_string();
17777        unknown_update.arguments = None;
17778        unknown_update.workflow_update_id = Some("update-unknown".to_string());
17779        unknown_update.update_name = Some("missing".to_string());
17780        let error = worker
17781            .execute_workflow_task(unknown_update)
17782            .expect_err("codec must precede fail_update shortcut");
17783        assert!(error.to_string().contains("unsupported_payload_codec"));
17784
17785        let unknown_query: QueryTask = serde_json::from_value(json!({
17786            "query_task_id": "query-unknown",
17787            "workflow_type": "missing",
17788            "query_name": "missing",
17789            "payload_codec": "json",
17790            "workflow_arguments": null,
17791            "query_arguments": null
17792        }))
17793        .expect("query task");
17794        let failure = worker
17795            .execute_query_task(unknown_query)
17796            .await
17797            .expect_err("codec must precede unknown query outcome");
17798        assert_eq!(failure.reason, "query_payload_decode_failed");
17799        assert!(failure.message.contains("unsupported_payload_codec"));
17800    }
17801
17802    #[tokio::test]
17803    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
17804        let client = Client::new("http://127.0.0.1:8080").expect("client");
17805        let worker = Worker::new(client, "rust-workers");
17806
17807        for event_type in ["SignalReceived", "SignalApplied"] {
17808            for (payload_field, codec) in [
17809                ("value", "json"),
17810                ("input", "unknown"),
17811                ("arguments", "json"),
17812            ] {
17813                let payload = json!({
17814                    "signal_name": "empty",
17815                    payload_field: {"codec": codec, "blob": null}
17816                });
17817                let workflow = workflow_task(
17818                    "missing",
17819                    vec![history_event(event_type, payload.clone())],
17820                    DEFAULT_CODEC,
17821                );
17822                let error = worker
17823                    .execute_workflow_task(workflow)
17824                    .expect_err("signal payload codec must precede unknown workflow outcome");
17825                assert!(
17826                    error.to_string().contains("unsupported_payload_codec"),
17827                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
17828                );
17829
17830                let query: QueryTask = serde_json::from_value(json!({
17831                    "query_task_id": format!("query-{event_type}-{payload_field}"),
17832                    "workflow_type": "missing",
17833                    "query_name": "missing",
17834                    "payload_codec": DEFAULT_CODEC,
17835                    "workflow_arguments": null,
17836                    "query_arguments": null,
17837                    "history_events": [{
17838                        "event_type": event_type,
17839                        "payload": payload
17840                    }]
17841                }))
17842                .expect("query task");
17843                let failure = worker
17844                    .execute_query_task(query)
17845                    .await
17846                    .expect_err("signal payload codec must precede unknown query outcome");
17847                assert_eq!(
17848                    failure.reason, "query_payload_decode_failed",
17849                    "{event_type}.{payload_field} returned an unrelated query outcome"
17850                );
17851                assert!(
17852                    failure.message.contains("unsupported_payload_codec"),
17853                    "{event_type}.{payload_field} returned an unrelated query error: {}",
17854                    failure.message
17855                );
17856            }
17857        }
17858    }
17859
17860    #[test]
17861    fn workflow_context_schedules_activity_until_completion_is_in_history() {
17862        let ctx = WorkflowContext {
17863            state: Arc::new(Mutex::new(
17864                WorkflowState::new_with_identity(
17865                    Vec::new(),
17866                    Some("wf-parent".to_string()),
17867                    Some("run-parent".to_string()),
17868                    "rust-workers".to_string(),
17869                    DEFAULT_CODEC.to_string(),
17870                    None,
17871                )
17872                .expect("workflow state"),
17873            )),
17874        };
17875
17876        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
17877        let mut task_context = TaskContext::from_waker(noop_waker_ref());
17878        assert!(matches!(
17879            call.as_mut().poll(&mut task_context),
17880            Poll::Pending
17881        ));
17882
17883        let commands = ctx.take_commands().expect("commands");
17884        assert_eq!(commands[0]["type"], "schedule_activity");
17885        assert_eq!(commands[0]["activity_type"], "hello.activity");
17886    }
17887
17888    #[test]
17889    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
17890        let ctx = workflow_context(Vec::new());
17891        let options = ActivityOptions::new()
17892            .task_queue("payments")
17893            .retry_policy(
17894                ActivityRetryPolicy::new(4)
17895                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
17896                    .non_retryable_error_type("ValidationError"),
17897            )
17898            .start_to_close_timeout(Duration::from_secs(120))
17899            .schedule_to_start_timeout(Duration::from_secs(10))
17900            .schedule_to_close_timeout(Duration::from_secs(300))
17901            .heartbeat_timeout(Duration::from_secs(15));
17902        let mut call = Box::pin(ctx.activity_with_options(
17903            "charge-card",
17904            options,
17905            json!([{"order_id": "o-1"}]),
17906        ));
17907        let mut task_context = TaskContext::from_waker(noop_waker_ref());
17908
17909        assert!(matches!(
17910            call.as_mut().poll(&mut task_context),
17911            Poll::Pending
17912        ));
17913        assert!(matches!(
17914            call.as_mut().poll(&mut task_context),
17915            Poll::Pending
17916        ));
17917
17918        let commands = ctx.take_commands().expect("activity command");
17919        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
17920        assert_eq!(commands[0]["queue"], "payments");
17921        assert_eq!(
17922            commands[0]["retry_policy"],
17923            json!({
17924                "max_attempts": 4,
17925                "backoff_seconds": [1, 3, 9],
17926                "non_retryable_error_types": ["ValidationError"],
17927            })
17928        );
17929        assert_eq!(commands[0]["start_to_close_timeout"], 120);
17930        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
17931        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
17932        assert_eq!(commands[0]["heartbeat_timeout"], 15);
17933    }
17934
17935    #[test]
17936    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
17937        let ctx = workflow_context(Vec::new());
17938        let options = ActivityOptions::new().retry_policy(
17939            ActivityRetryPolicy::new(3)
17940                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
17941        );
17942        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
17943        let mut task_context = TaskContext::from_waker(noop_waker_ref());
17944
17945        assert!(matches!(
17946            call.as_mut().poll(&mut task_context),
17947            Poll::Pending
17948        ));
17949        assert_eq!(
17950            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
17951            json!([1, 2])
17952        );
17953    }
17954
17955    #[test]
17956    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
17957        let cases = [
17958            (
17959                ActivityOptions::new().task_queue("  "),
17960                ActivityOptionsErrorKind::EmptyTaskQueue,
17961            ),
17962            (
17963                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
17964                ActivityOptionsErrorKind::EmptyRetryPolicy,
17965            ),
17966            (
17967                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
17968                ActivityOptionsErrorKind::InvalidMaxAttempts,
17969            ),
17970            (
17971                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
17972                    max_attempts: None,
17973                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
17974                    non_retryable_error_types: Vec::new(),
17975                }),
17976                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
17977            ),
17978            (
17979                ActivityOptions::new().retry_policy(
17980                    ActivityRetryPolicy::new(2)
17981                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
17982                ),
17983                ActivityOptionsErrorKind::TooManyBackoffIntervals,
17984            ),
17985            (
17986                ActivityOptions::new().retry_policy(
17987                    ActivityRetryPolicy::new(2).exponential_backoff(
17988                        Duration::from_secs(1),
17989                        0,
17990                        None,
17991                    ),
17992                ),
17993                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
17994            ),
17995            (
17996                ActivityOptions::new()
17997                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
17998                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
17999            ),
18000            (
18001                ActivityOptions::new().retry_policy(
18002                    ActivityRetryPolicy::new(10_002).exponential_backoff(
18003                        Duration::from_secs(1),
18004                        1,
18005                        None,
18006                    ),
18007                ),
18008                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18009            ),
18010            (
18011                ActivityOptions::new().retry_policy(
18012                    ActivityRetryPolicy::new(2)
18013                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18014                ),
18015                ActivityOptionsErrorKind::BackoffOverflow,
18016            ),
18017        ];
18018
18019        for (options, expected_kind) in cases {
18020            let ctx = workflow_context(Vec::new());
18021            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18022            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18023            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18024                call.as_mut().poll(&mut task_context)
18025            else {
18026                panic!("expected typed activity validation error");
18027            };
18028            assert_eq!(error.kind, expected_kind);
18029            assert!(ctx.take_commands().expect("commands").is_empty());
18030        }
18031    }
18032
18033    #[test]
18034    fn activity_options_validate_positive_and_ordered_timeouts() {
18035        let zero_timeout_cases = [
18036            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18037            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18038            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18039            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18040        ];
18041        for options in zero_timeout_cases {
18042            assert_eq!(
18043                options.validate().expect_err("zero timeout").kind,
18044                ActivityOptionsErrorKind::TimeoutNotPositive
18045            );
18046        }
18047
18048        let ordering_cases = [
18049            ActivityOptions::new()
18050                .heartbeat_timeout(Duration::from_secs(11))
18051                .start_to_close_timeout(Duration::from_secs(10)),
18052            ActivityOptions::new()
18053                .start_to_close_timeout(Duration::from_secs(31))
18054                .schedule_to_close_timeout(Duration::from_secs(30)),
18055            ActivityOptions::new()
18056                .schedule_to_start_timeout(Duration::from_secs(31))
18057                .schedule_to_close_timeout(Duration::from_secs(30)),
18058        ];
18059        for options in ordering_cases {
18060            assert_eq!(
18061                options.validate().expect_err("timeout order").kind,
18062                ActivityOptionsErrorKind::TimeoutOrder
18063            );
18064        }
18065
18066        assert_eq!(
18067            ActivityOptions::new()
18068                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18069                .validate()
18070                .expect_err("protocol integer overflow")
18071                .kind,
18072            ActivityOptionsErrorKind::TimeoutOverflow
18073        );
18074    }
18075
18076    #[test]
18077    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18078        let ctx = workflow_context(completed_retry_activity_history());
18079        let mut call =
18080            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18081        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18082
18083        assert!(matches!(
18084            call.as_mut().poll(&mut task_context),
18085            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18086        ));
18087        assert!(ctx.take_commands().expect("commands").is_empty());
18088        ctx.ensure_history_consumed().expect("history consumed");
18089    }
18090
18091    #[test]
18092    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18093        let mut options = retry_activity_options();
18094        options
18095            .retry_policy
18096            .as_mut()
18097            .expect("retry policy")
18098            .non_retryable_error_types
18099            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18100
18101        let new_ctx = workflow_context(Vec::new());
18102        let mut new_call =
18103            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18104        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18105        assert!(matches!(
18106            new_call.as_mut().poll(&mut task_context),
18107            Poll::Pending
18108        ));
18109        let commands = new_ctx.take_commands().expect("commands");
18110        assert_eq!(commands.len(), 1);
18111        assert_eq!(
18112            commands[0]["retry_policy"]["non_retryable_error_types"],
18113            json!(["PermanentError"])
18114        );
18115
18116        let replay_ctx = workflow_context(completed_retry_activity_history());
18117        let mut replay_call =
18118            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18119        assert!(matches!(
18120            replay_call.as_mut().poll(&mut task_context),
18121            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18122        ));
18123        assert!(replay_ctx.take_commands().expect("commands").is_empty());
18124        replay_ctx
18125            .ensure_history_consumed()
18126            .expect("history consumed");
18127    }
18128
18129    #[test]
18130    fn replayed_intermediate_retry_remains_pending_across_restarts() {
18131        let history = completed_retry_activity_history()
18132            .into_iter()
18133            .take(3)
18134            .collect::<Vec<_>>();
18135
18136        for _restart in 0..2 {
18137            let ctx = workflow_context(history.clone());
18138            let mut call =
18139                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18140            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18141            assert!(matches!(
18142                call.as_mut().poll(&mut task_context),
18143                Poll::Pending
18144            ));
18145            assert!(ctx.take_commands().expect("commands").is_empty());
18146        }
18147    }
18148
18149    #[test]
18150    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18151        let mut changed_queue = retry_activity_options();
18152        changed_queue.task_queue = Some("different-queue".to_string());
18153
18154        let mut changed_max_attempts = retry_activity_options();
18155        let retry_policy = changed_max_attempts
18156            .retry_policy
18157            .as_mut()
18158            .expect("retry policy");
18159        retry_policy.max_attempts = Some(4);
18160
18161        let mut changed_backoff = retry_activity_options();
18162        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18163        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18164            Duration::from_secs(3),
18165            Duration::from_secs(4),
18166        ]));
18167
18168        let mut changed_non_retryable_types = retry_activity_options();
18169        let retry_policy = changed_non_retryable_types
18170            .retry_policy
18171            .as_mut()
18172            .expect("retry policy");
18173        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18174
18175        let mut changed_start_to_close = retry_activity_options();
18176        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18177        let mut changed_schedule_to_start = retry_activity_options();
18178        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18179        let mut changed_schedule_to_close = retry_activity_options();
18180        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18181        let mut changed_heartbeat = retry_activity_options();
18182        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18183
18184        let cases = [
18185            (changed_queue, "activity_task_queue_mismatch"),
18186            (changed_max_attempts, "activity_retry_policy_mismatch"),
18187            (changed_backoff, "activity_retry_policy_mismatch"),
18188            (
18189                changed_non_retryable_types,
18190                "activity_retry_policy_mismatch",
18191            ),
18192            (changed_start_to_close, "activity_retry_policy_mismatch"),
18193            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18194            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18195            (changed_heartbeat, "activity_retry_policy_mismatch"),
18196        ];
18197
18198        for (options, expected_reason) in cases {
18199            let ctx = workflow_context(completed_retry_activity_history());
18200            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18201            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18202            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18203                call.as_mut().poll(&mut task_context)
18204            else {
18205                panic!("changed activity options must fail replay");
18206            };
18207            assert_eq!(failure.reason, expected_reason);
18208            assert_eq!(failure.sequence, Some(1));
18209            assert!(ctx.take_commands().expect("commands").is_empty());
18210        }
18211    }
18212
18213    #[test]
18214    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18215        let cases = [
18216            (
18217                "execution_mode",
18218                json!("local"),
18219                "activity_execution_mode_mismatch",
18220            ),
18221            (
18222                "snapshot_version",
18223                json!(2),
18224                "activity_retry_policy_mismatch",
18225            ),
18226        ];
18227
18228        for (field, value, expected_reason) in cases {
18229            let mut history = completed_retry_activity_history();
18230            let activity = history[0].payload["activity"]
18231                .as_object_mut()
18232                .expect("activity snapshot");
18233            if field == "execution_mode" {
18234                activity.insert(field.to_string(), value);
18235            } else {
18236                activity["retry_policy"]
18237                    .as_object_mut()
18238                    .expect("retry snapshot")
18239                    .insert(field.to_string(), value);
18240            }
18241
18242            let ctx = workflow_context(history);
18243            let mut call =
18244                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18245            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18246            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18247                call.as_mut().poll(&mut task_context)
18248            else {
18249                panic!("changed {field} must fail replay");
18250            };
18251            assert_eq!(failure.reason, expected_reason);
18252            assert_eq!(failure.sequence, Some(1));
18253            assert!(ctx.take_commands().expect("commands").is_empty());
18254        }
18255    }
18256
18257    #[test]
18258    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18259        let mut history = completed_retry_activity_history();
18260        let activity = history[0].payload["activity"]
18261            .as_object_mut()
18262            .expect("activity snapshot");
18263        activity.remove("execution_mode");
18264        activity.remove("retry_policy");
18265
18266        let mut current = retry_activity_options();
18267        current.start_to_close_timeout = Some(Duration::from_secs(45));
18268        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18269        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18270        current.heartbeat_timeout = Some(Duration::from_secs(12));
18271
18272        let ctx = workflow_context(history);
18273        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18274        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18275        assert!(matches!(
18276            call.as_mut().poll(&mut task_context),
18277            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18278        ));
18279        assert!(ctx.take_commands().expect("commands").is_empty());
18280        ctx.ensure_history_consumed().expect("history consumed");
18281    }
18282
18283    #[test]
18284    fn terminal_activity_failed_after_start_returns_typed_failure() {
18285        let history = vec![
18286            history_event(
18287                "ActivityScheduled",
18288                json!({
18289                    "sequence": 1,
18290                    "activity_type": "flaky",
18291                    "activity_execution_id": "act-terminal",
18292                    "activity": {
18293                        "id": "act-terminal",
18294                        "sequence": 1,
18295                        "type": "flaky",
18296                        "queue": "critical-activities",
18297                        "retry_policy": {
18298                            "snapshot_version": 1,
18299                            "max_attempts": 3,
18300                            "backoff_seconds": [2, 4],
18301                            "non_retryable_error_types": ["PermanentError"]
18302                        }
18303                    }
18304                }),
18305            ),
18306            history_event(
18307                "ActivityStarted",
18308                json!({
18309                    "sequence": 1,
18310                    "activity_type": "flaky",
18311                    "activity_execution_id": "act-terminal",
18312                    "activity_attempt_id": "attempt-1",
18313                    "attempt_number": 1
18314                }),
18315            ),
18316            history_event(
18317                "ActivityFailed",
18318                json!({
18319                    "sequence": 1,
18320                    "activity_type": "flaky",
18321                    "activity_execution_id": "act-terminal",
18322                    "activity_attempt_id": "attempt-1",
18323                    "attempt_number": 1,
18324                    "failure_id": "failure-terminal",
18325                    "failure_category": "activity",
18326                    "exception_type": "PermanentError",
18327                    "message": "cannot retry",
18328                    "non_retryable": true
18329                }),
18330            ),
18331        ];
18332        let ctx = workflow_context(history);
18333        let mut call =
18334            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18335        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18336
18337        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18338            call.as_mut().poll(&mut task_context)
18339        else {
18340            panic!("terminal ActivityFailed must settle the activity future");
18341        };
18342        assert_eq!(failure.kind, ActivityFailureKind::Failed);
18343        assert_eq!(
18344            failure.activity_execution_id.as_deref(),
18345            Some("act-terminal")
18346        );
18347        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18348        assert!(failure.non_retryable);
18349        assert!(ctx.take_commands().expect("commands").is_empty());
18350        ctx.ensure_history_consumed().expect("history consumed");
18351    }
18352
18353    #[test]
18354    fn activity_terminal_events_return_machine_readable_failures() {
18355        let cases = [
18356            (
18357                "ActivityFailed",
18358                json!({
18359                    "sequence": 1,
18360                    "activity_type": "charge-card",
18361                    "activity_execution_id": "act-1",
18362                    "activity_attempt_id": "attempt-2",
18363                    "attempt_number": 2,
18364                    "failure_id": "failure-1",
18365                    "failure_category": "activity",
18366                    "exception_type": "PaymentDeclined",
18367                    "exception_class": "payments.PaymentDeclined",
18368                    "message": "card declined",
18369                    "non_retryable": true
18370                }),
18371                ActivityFailureKind::Failed,
18372                "activity",
18373            ),
18374            (
18375                "ActivityCancelled",
18376                json!({
18377                    "sequence": 1,
18378                    "activity_type": "charge-card",
18379                    "activity_execution_id": "act-1",
18380                    "activity_attempt_id": "attempt-1"
18381                }),
18382                ActivityFailureKind::Cancelled,
18383                "cancelled",
18384            ),
18385        ];
18386
18387        for (event_type, payload, expected_kind, expected_reason) in cases {
18388            let ctx = workflow_context(vec![history_event(event_type, payload)]);
18389            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18390            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18391            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18392                call.as_mut().poll(&mut task_context)
18393            else {
18394                panic!("expected terminal activity failure");
18395            };
18396            assert_eq!(failure.kind, expected_kind);
18397            assert_eq!(failure.reason, expected_reason);
18398            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18399            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18400        }
18401    }
18402
18403    #[test]
18404    fn every_activity_timeout_class_is_typed() {
18405        for timeout_kind in [
18406            "start_to_close",
18407            "schedule_to_start",
18408            "schedule_to_close",
18409            "heartbeat",
18410        ] {
18411            let ctx = workflow_context(vec![history_event(
18412                "ActivityTimedOut",
18413                json!({
18414                    "sequence": 1,
18415                    "activity_type": "slow",
18416                    "activity_execution_id": "act-timeout",
18417                    "activity_attempt_id": "attempt-timeout",
18418                    "failure_category": "timeout",
18419                    "timeout_kind": timeout_kind,
18420                    "message": "deadline expired"
18421                }),
18422            )]);
18423            let mut call = Box::pin(ctx.activity("slow", json!([])));
18424            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18425            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18426                call.as_mut().poll(&mut task_context)
18427            else {
18428                panic!("expected timeout failure");
18429            };
18430            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18431            assert_eq!(failure.reason, timeout_kind);
18432            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18433            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18434        }
18435    }
18436
18437    #[test]
18438    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18439        let ctx = workflow_context(Vec::new());
18440        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18441        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18442
18443        assert!(matches!(
18444            sleep.as_mut().poll(&mut task_context),
18445            Poll::Pending
18446        ));
18447        assert!(matches!(
18448            sleep.as_mut().poll(&mut task_context),
18449            Poll::Pending
18450        ));
18451
18452        let commands = ctx.take_commands().expect("timer command");
18453        assert_eq!(
18454            commands,
18455            vec![json!({
18456                "type": "start_timer",
18457                "delay_seconds": 2,
18458            })]
18459        );
18460    }
18461
18462    #[test]
18463    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18464        let history = vec![
18465            history_event(
18466                "TimerScheduled",
18467                json!({
18468                    "sequence": 1,
18469                    "timer_id": "timer-1",
18470                    "delay_seconds": 5,
18471                    "fire_at": "2026-07-11T12:00:05Z",
18472                }),
18473            ),
18474            history_event(
18475                "TimerFired",
18476                json!({
18477                    "sequence": 1,
18478                    "timer_id": "timer-1",
18479                    "delay_seconds": 5,
18480                    "fire_at": "2026-07-11T12:00:05Z",
18481                    "fired_at": "2026-07-11T12:00:05Z",
18482                }),
18483            ),
18484        ];
18485
18486        for _restart in 0..2 {
18487            let ctx = workflow_context(history.clone());
18488            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18489            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18490            assert!(matches!(
18491                sleep.as_mut().poll(&mut task_context),
18492                Poll::Ready(Ok(()))
18493            ));
18494            assert!(ctx.take_commands().expect("commands").is_empty());
18495            ctx.ensure_history_consumed().expect("history consumed");
18496        }
18497    }
18498
18499    #[test]
18500    fn workflow_sleep_rejects_changed_delay_during_replay() {
18501        let ctx = workflow_context(vec![
18502            history_event(
18503                "TimerScheduled",
18504                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18505            ),
18506            history_event(
18507                "TimerFired",
18508                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18509            ),
18510        ]);
18511        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18512        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18513
18514        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18515            sleep.as_mut().poll(&mut task_context)
18516        else {
18517            panic!("changed timer delay must be rejected");
18518        };
18519        assert_eq!(failure.reason, "timer_delay_mismatch");
18520        assert_eq!(failure.sequence, Some(1));
18521    }
18522
18523    #[test]
18524    fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18525        let ctx = workflow_context(Vec::new());
18526        let mut wait = Box::pin(
18527            ctx.wait_condition(
18528                ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18529                    .timeout(Duration::from_millis(60_001)),
18530                || Ok(false),
18531            ),
18532        );
18533        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18534
18535        assert!(matches!(
18536            wait.as_mut().poll(&mut task_context),
18537            Poll::Pending
18538        ));
18539        assert!(matches!(
18540            wait.as_mut().poll(&mut task_context),
18541            Poll::Pending
18542        ));
18543        assert_eq!(
18544            ctx.take_commands().expect("condition command"),
18545            vec![json!({
18546                "type": "open_condition_wait",
18547                "condition_wait_occurrence_id": "rust:condition-wait:0",
18548                "condition_key": "approval.ready",
18549                "condition_definition_fingerprint": "sha256:approval-v1",
18550                "timeout_seconds": 61,
18551            })]
18552        );
18553    }
18554
18555    #[test]
18556    fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18557        let ctx = workflow_context(Vec::new());
18558        let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18559        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18560        assert!(matches!(
18561            satisfied.as_mut().poll(&mut task_context),
18562            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18563        ));
18564
18565        let mut timed_out = Box::pin(wait_condition!(
18566            ctx,
18567            "no-wait",
18568            timeout: Duration::ZERO,
18569            || Ok(false),
18570        ));
18571        assert!(matches!(
18572            timed_out.as_mut().poll(&mut task_context),
18573            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18574        ));
18575        assert!(ctx.take_commands().expect("commands").is_empty());
18576    }
18577
18578    #[test]
18579    fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18580        let signal_history = vec![
18581            history_event(
18582                "ConditionWaitOpened",
18583                json!({
18584                    "sequence": 4,
18585                    "condition_wait_id": "condition:4",
18586                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18587                    "condition_key": "approval",
18588                    "condition_definition_fingerprint": "sha256:approval-v1",
18589                    "timeout_seconds": 30,
18590                }),
18591            ),
18592            history_event(
18593                "SignalReceived",
18594                json!({
18595                    "workflow_sequence": 4,
18596                    "signal_name": "approve",
18597                    "arguments": fixture_envelope(json!(["Ada"])),
18598                }),
18599            ),
18600        ];
18601        for _worker_before_or_after_restart in 0..2 {
18602            let ctx = workflow_context(signal_history.clone());
18603            let predicate_ctx = ctx.clone();
18604            let mut wait = Box::pin(
18605                ctx.wait_condition(
18606                    ConditionWaitOptions::new("approval", "sha256:approval-v1")
18607                        .timeout(Duration::from_secs(30)),
18608                    move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18609                ),
18610            );
18611            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18612            assert!(matches!(
18613                wait.as_mut().poll(&mut task_context),
18614                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18615            ));
18616            assert!(ctx.take_commands().expect("commands").is_empty());
18617            ctx.ensure_history_consumed().expect("condition consumed");
18618        }
18619
18620        let update_history = vec![
18621            history_event(
18622                "ConditionWaitOpened",
18623                json!({
18624                    "sequence": 7,
18625                    "condition_wait_id": "condition:7",
18626                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18627                    "condition_key": "update-approval",
18628                    "condition_definition_fingerprint": "sha256:update-approval-v1",
18629                }),
18630            ),
18631            history_event(
18632                "UpdateApplied",
18633                json!({
18634                    "sequence": 7,
18635                    "update_id": "update-1",
18636                    "update_name": "approve",
18637                    "arguments": fixture_envelope(json!([true])),
18638                }),
18639            ),
18640        ];
18641        let ctx = workflow_context(update_history);
18642        let predicate_ctx = ctx.clone();
18643        let mut wait = Box::pin(ctx.wait_condition(
18644            ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18645            move || {
18646                Ok(predicate_ctx
18647                    .updates("approve")?
18648                    .first()
18649                    .and_then(|arguments| arguments.first())
18650                    .and_then(Value::as_bool)
18651                    == Some(true))
18652            },
18653        ));
18654        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18655        assert!(matches!(
18656            wait.as_mut().poll(&mut task_context),
18657            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18658        ));
18659        assert!(ctx.take_commands().expect("commands").is_empty());
18660        ctx.ensure_history_consumed().expect("condition consumed");
18661    }
18662
18663    #[test]
18664    fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18665        let open_history = vec![
18666            history_event(
18667                "ConditionWaitOpened",
18668                json!({
18669                    "sequence": 3,
18670                    "condition_wait_id": "condition:3",
18671                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18672                    "condition_key": "two-votes",
18673                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18674                    "timeout_seconds": 120,
18675                }),
18676            ),
18677            history_event(
18678                "SignalReceived",
18679                json!({
18680                    "workflow_sequence": 3,
18681                    "signal_name": "vote",
18682                    "arguments": fixture_envelope(json!(["first"])),
18683                }),
18684            ),
18685        ];
18686        for _worker_before_or_after_restart in 0..2 {
18687            let ctx = workflow_context(open_history.clone());
18688            let predicate_ctx = ctx.clone();
18689            let mut wait = Box::pin(
18690                ctx.wait_condition(
18691                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18692                        .timeout(Duration::from_secs(120)),
18693                    move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18694                ),
18695            );
18696            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18697            assert!(matches!(
18698                wait.as_mut().poll(&mut task_context),
18699                Poll::Pending
18700            ));
18701            assert_eq!(
18702                ctx.take_commands().expect("reopened condition"),
18703                vec![json!({
18704                    "type": "open_condition_wait",
18705                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18706                    "condition_key": "two-votes",
18707                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18708                    "timeout_seconds": 120,
18709                })]
18710            );
18711        }
18712
18713        let satisfied_ctx = workflow_context(vec![
18714            history_event(
18715                "ConditionWaitOpened",
18716                json!({
18717                    "sequence": 5,
18718                    "condition_wait_id": "condition:5",
18719                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18720                    "condition_key": "approval",
18721                    "condition_definition_fingerprint": "sha256:approval-v1",
18722                }),
18723            ),
18724            history_event(
18725                "ConditionWaitSatisfied",
18726                json!({
18727                    "sequence": 5,
18728                    "condition_wait_id": "condition:5",
18729                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18730                    "condition_key": "approval",
18731                    "condition_definition_fingerprint": "sha256:approval-v1",
18732                }),
18733            ),
18734        ]);
18735        let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18736            ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18737            || Ok(false),
18738        ));
18739        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18740        assert!(matches!(
18741            satisfied.as_mut().poll(&mut task_context),
18742            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18743        ));
18744
18745        let timed_out_ctx = workflow_context(vec![
18746            history_event(
18747                "ConditionWaitOpened",
18748                json!({
18749                    "sequence": 8,
18750                    "condition_wait_id": "condition:8",
18751                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18752                    "condition_key": "approval-timeout",
18753                    "condition_definition_fingerprint": "sha256:approval-timeout-v1",
18754                    "timeout_seconds": 5,
18755                }),
18756            ),
18757            history_event(
18758                "TimerScheduled",
18759                json!({
18760                    "sequence": 9,
18761                    "timer_id": "condition-timer:9",
18762                    "timer_kind": "condition_timeout",
18763                    "condition_wait_id": "condition:8",
18764                    "delay_seconds": 5,
18765                }),
18766            ),
18767            history_event(
18768                "TimerFired",
18769                json!({
18770                    "sequence": 9,
18771                    "timer_id": "condition-timer:9",
18772                    "timer_kind": "condition_timeout",
18773                    "condition_wait_id": "condition:8",
18774                    "delay_seconds": 5,
18775                }),
18776            ),
18777        ]);
18778        let mut timed_out = Box::pin(
18779            timed_out_ctx.wait_condition(
18780                ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
18781                    .timeout(Duration::from_secs(5)),
18782                || Ok(true),
18783            ),
18784        );
18785        assert!(matches!(
18786            timed_out.as_mut().poll(&mut task_context),
18787            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18788        ));
18789    }
18790
18791    #[test]
18792    fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
18793        let history = vec![
18794            history_event(
18795                "ConditionWaitOpened",
18796                json!({
18797                    "sequence": 3,
18798                    "condition_wait_id": "condition:3",
18799                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18800                    "condition_key": "two-votes",
18801                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18802                }),
18803            ),
18804            history_event(
18805                "SignalReceived",
18806                json!({
18807                    "workflow_sequence": 3,
18808                    "signal_name": "vote",
18809                    "arguments": fixture_envelope(json!(["first"])),
18810                }),
18811            ),
18812            history_event(
18813                "ConditionWaitSatisfied",
18814                json!({
18815                    "sequence": 3,
18816                    "condition_wait_id": "condition:3",
18817                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18818                    "condition_key": "two-votes",
18819                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18820                }),
18821            ),
18822            history_event(
18823                "ConditionWaitOpened",
18824                json!({
18825                    "sequence": 5,
18826                    "condition_wait_id": "condition:5",
18827                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18828                    "condition_key": "two-votes",
18829                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18830                }),
18831            ),
18832            history_event(
18833                "SignalReceived",
18834                json!({
18835                    "workflow_sequence": 5,
18836                    "signal_name": "vote",
18837                    "arguments": fixture_envelope(json!(["second"])),
18838                }),
18839            ),
18840            history_event(
18841                "ConditionWaitSatisfied",
18842                json!({
18843                    "sequence": 5,
18844                    "condition_wait_id": "condition:5",
18845                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18846                    "condition_key": "two-votes",
18847                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18848                }),
18849            ),
18850        ];
18851        for _cold_worker_or_restart in 0..2 {
18852            let ctx = workflow_context(history.clone());
18853            let predicate_ctx = ctx.clone();
18854            let mut wait = Box::pin(ctx.wait_condition(
18855                ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
18856                move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18857            ));
18858            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18859
18860            assert!(matches!(
18861                wait.as_mut().poll(&mut task_context),
18862                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18863            ));
18864            assert!(ctx.take_commands().expect("commands").is_empty());
18865            ctx.ensure_history_consumed()
18866                .expect("every physical wait-open is consumed");
18867        }
18868    }
18869
18870    #[test]
18871    fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
18872        let history = vec![
18873            history_event(
18874                "ConditionWaitOpened",
18875                json!({
18876                    "sequence": 3,
18877                    "condition_wait_id": "condition:3",
18878                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18879                    "condition_key": "approved",
18880                    "condition_definition_fingerprint": "sha256:approved-v1",
18881                }),
18882            ),
18883            history_event(
18884                "UpdateApplied",
18885                json!({
18886                    "sequence": 3,
18887                    "update_id": "update-1",
18888                    "update_name": "approve",
18889                    "arguments": fixture_envelope(json!([false])),
18890                }),
18891            ),
18892            history_event(
18893                "ConditionWaitOpened",
18894                json!({
18895                    "sequence": 5,
18896                    "condition_wait_id": "condition:5",
18897                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18898                    "condition_key": "approved",
18899                    "condition_definition_fingerprint": "sha256:approved-v1",
18900                }),
18901            ),
18902            history_event(
18903                "UpdateApplied",
18904                json!({
18905                    "sequence": 5,
18906                    "update_id": "update-2",
18907                    "update_name": "approve",
18908                    "arguments": fixture_envelope(json!([true])),
18909                }),
18910            ),
18911        ];
18912
18913        for _cold_worker_or_restart in 0..2 {
18914            let ctx = workflow_context(history.clone());
18915            let predicate_ctx = ctx.clone();
18916            let mut wait = Box::pin(ctx.wait_condition(
18917                ConditionWaitOptions::new("approved", "sha256:approved-v1"),
18918                move || {
18919                    Ok(predicate_ctx
18920                        .updates("approve")?
18921                        .last()
18922                        .and_then(|arguments| arguments.first())
18923                        .and_then(Value::as_bool)
18924                        == Some(true))
18925                },
18926            ));
18927            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18928
18929            assert!(matches!(
18930                wait.as_mut().poll(&mut task_context),
18931                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18932            ));
18933            assert!(ctx.take_commands().expect("commands").is_empty());
18934            ctx.ensure_history_consumed()
18935                .expect("every update-driven reopen is consumed");
18936        }
18937    }
18938
18939    #[test]
18940    fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
18941        for (first_key, first_fingerprint, second_key, second_fingerprint) in [
18942            ("shared", "sha256:first", "shared", "sha256:second"),
18943            ("first", "sha256:shared", "second", "sha256:shared"),
18944            ("shared", "sha256:shared", "shared", "sha256:shared"),
18945            ("first", "sha256:first", "second", "sha256:second"),
18946        ] {
18947            let history = vec![
18948                history_event(
18949                    "ConditionWaitOpened",
18950                    json!({
18951                        "sequence": 3,
18952                        "condition_wait_id": "condition:3",
18953                        "condition_wait_occurrence_id": "rust:condition-wait:0",
18954                        "condition_key": first_key,
18955                        "condition_definition_fingerprint": first_fingerprint,
18956                    }),
18957                ),
18958                history_event(
18959                    "ConditionWaitSatisfied",
18960                    json!({
18961                        "sequence": 3,
18962                        "condition_wait_id": "condition:3",
18963                        "condition_wait_occurrence_id": "rust:condition-wait:0",
18964                        "condition_key": first_key,
18965                        "condition_definition_fingerprint": first_fingerprint,
18966                    }),
18967                ),
18968                history_event(
18969                    "ConditionWaitOpened",
18970                    json!({
18971                        "sequence": 4,
18972                        "condition_wait_id": "condition:4",
18973                        "condition_wait_occurrence_id": "rust:condition-wait:1",
18974                        "condition_key": second_key,
18975                        "condition_definition_fingerprint": second_fingerprint,
18976                    }),
18977                ),
18978                history_event(
18979                    "ConditionWaitSatisfied",
18980                    json!({
18981                        "sequence": 4,
18982                        "condition_wait_id": "condition:4",
18983                        "condition_wait_occurrence_id": "rust:condition-wait:1",
18984                        "condition_key": second_key,
18985                        "condition_definition_fingerprint": second_fingerprint,
18986                    }),
18987                ),
18988            ];
18989            for _cold_worker_or_restart in 0..2 {
18990                let ctx = workflow_context(history.clone());
18991                let mut task_context = TaskContext::from_waker(noop_waker_ref());
18992                let mut first = Box::pin(ctx.wait_condition(
18993                    ConditionWaitOptions::new(first_key, first_fingerprint),
18994                    || Ok(false),
18995                ));
18996                assert!(matches!(
18997                    first.as_mut().poll(&mut task_context),
18998                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18999                ));
19000
19001                let mut second = Box::pin(ctx.wait_condition(
19002                    ConditionWaitOptions::new(second_key, second_fingerprint),
19003                    || Ok(false),
19004                ));
19005                assert!(matches!(
19006                    second.as_mut().poll(&mut task_context),
19007                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19008                ));
19009                assert!(ctx.take_commands().expect("commands").is_empty());
19010                ctx.ensure_history_consumed()
19011                    .expect("each authored wait consumes one occurrence");
19012            }
19013        }
19014    }
19015
19016    #[test]
19017    fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19018        fn worker() -> Worker {
19019            let client = Client::new("http://127.0.0.1:8080").expect("client");
19020            let mut worker = Worker::new(client, "rust-workers");
19021            worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19022                let mut outcomes = Vec::new();
19023                for _ in 0..2 {
19024                    outcomes.push(
19025                        ctx.wait_condition(
19026                            ConditionWaitOptions::new("shared", "sha256:shared"),
19027                            || Ok(false),
19028                        )
19029                        .await?,
19030                    );
19031                }
19032                Ok(json!(outcomes))
19033            });
19034            worker
19035        }
19036
19037        let task = workflow_task(
19038            "rust.condition-loop",
19039            vec![
19040                history_event(
19041                    "ConditionWaitOpened",
19042                    json!({
19043                        "sequence": 1,
19044                        "condition_wait_id": "condition:1",
19045                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19046                        "condition_key": "shared",
19047                        "condition_definition_fingerprint": "sha256:shared",
19048                    }),
19049                ),
19050                history_event(
19051                    "ConditionWaitSatisfied",
19052                    json!({
19053                        "sequence": 1,
19054                        "condition_wait_id": "condition:1",
19055                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19056                        "condition_key": "shared",
19057                        "condition_definition_fingerprint": "sha256:shared",
19058                    }),
19059                ),
19060                history_event(
19061                    "ConditionWaitOpened",
19062                    json!({
19063                        "sequence": 2,
19064                        "condition_wait_id": "condition:2",
19065                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19066                        "condition_key": "shared",
19067                        "condition_definition_fingerprint": "sha256:shared",
19068                    }),
19069                ),
19070                history_event(
19071                    "ConditionWaitSatisfied",
19072                    json!({
19073                        "sequence": 2,
19074                        "condition_wait_id": "condition:2",
19075                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19076                        "condition_key": "shared",
19077                        "condition_definition_fingerprint": "sha256:shared",
19078                    }),
19079                ),
19080            ],
19081            DEFAULT_CODEC,
19082        );
19083
19084        for _cold_worker_or_restart in 0..2 {
19085            let commands = worker()
19086                .execute_workflow_task(task.clone())
19087                .expect("adjacent loop waits replay deterministically");
19088            assert_eq!(commands.len(), 1);
19089            assert_eq!(commands[0]["type"], "complete_workflow");
19090            assert_eq!(
19091                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19092                json!(["satisfied", "satisfied"])
19093            );
19094        }
19095    }
19096
19097    #[test]
19098    fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19099        let history = vec![history_event(
19100            "ConditionWaitOpened",
19101            json!({
19102                "sequence": 12,
19103                "condition_wait_id": "condition:12",
19104                "condition_wait_occurrence_id": "rust:condition-wait:0",
19105                "condition_key": "approval",
19106                "condition_definition_fingerprint": "sha256:approval-v1",
19107                "timeout_seconds": 30,
19108            }),
19109        )];
19110        for (options, expected_reason) in [
19111            (
19112                ConditionWaitOptions::new("changed", "sha256:approval-v1")
19113                    .timeout(Duration::from_secs(30)),
19114                "condition_wait_key_mismatch",
19115            ),
19116            (
19117                ConditionWaitOptions::new("approval", "sha256:approval-v2")
19118                    .timeout(Duration::from_secs(30)),
19119                "condition_wait_predicate_mismatch",
19120            ),
19121            (
19122                ConditionWaitOptions::new("approval", "sha256:approval-v1")
19123                    .timeout(Duration::from_secs(29)),
19124                "condition_wait_timeout_mismatch",
19125            ),
19126        ] {
19127            let ctx = workflow_context(history.clone());
19128            let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19129            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19130            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19131                wait.as_mut().poll(&mut task_context)
19132            else {
19133                panic!("changed condition definition must fail replay");
19134            };
19135            assert_eq!(failure.reason, expected_reason);
19136            assert_eq!(failure.sequence, Some(12));
19137        }
19138    }
19139
19140    #[test]
19141    fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19142        let error = WorkflowState::new(
19143            vec![history_event(
19144                "ConditionWaitOpened",
19145                json!({
19146                    "sequence": 12,
19147                    "condition_wait_id": "condition:12",
19148                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19149                    "condition_key": "approval",
19150                }),
19151            )],
19152            "rust-workers".to_string(),
19153            DEFAULT_CODEC.to_string(),
19154            None,
19155        )
19156        .expect_err("condition history without a predicate fingerprint must fail");
19157
19158        assert!(matches!(
19159            error,
19160            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19161                if reason == "condition_wait_predicate_fingerprint_missing"
19162        ));
19163    }
19164
19165    #[test]
19166    fn condition_wait_history_requires_authored_occurrence_identity() {
19167        let error = WorkflowState::new(
19168            vec![history_event(
19169                "ConditionWaitOpened",
19170                json!({
19171                    "sequence": 12,
19172                    "condition_wait_id": "condition:12",
19173                    "condition_key": "approval",
19174                    "condition_definition_fingerprint": "sha256:approval-v1",
19175                }),
19176            )],
19177            "rust-workers".to_string(),
19178            DEFAULT_CODEC.to_string(),
19179            None,
19180        )
19181        .expect_err("condition history without occurrence identity must fail");
19182
19183        assert!(matches!(
19184            error,
19185            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19186                if reason == "condition_wait_occurrence_id_missing"
19187        ));
19188    }
19189
19190    #[test]
19191    fn typed_search_attribute_updates_validate_emit_and_replay() {
19192        let update = SearchAttributeUpdate::new()
19193            .keyword("OrderStatus", " waiting ")
19194            .expect("keyword")
19195            .int("Attempt", 3)
19196            .expect("int")
19197            .bool("Escalated", false)
19198            .expect("bool")
19199            .keyword_list("Regions", ["us-east", "eu-west"])
19200            .expect("list")
19201            .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19202            .expect("datetime")
19203            .delete("LegacyStatus")
19204            .expect("delete");
19205        let ctx = workflow_context(Vec::new());
19206        ctx.upsert_search_attributes(update.clone())
19207            .expect("typed update");
19208        assert_eq!(
19209            ctx.take_commands().expect("search-attribute command"),
19210            vec![json!({
19211                "type": "upsert_search_attributes",
19212                "attributes": {
19213                    "Attempt": 3,
19214                    "Escalated": false,
19215                    "LegacyStatus": null,
19216                    "OrderStatus": "waiting",
19217                    "Regions": ["us-east", "eu-west"],
19218                    "UpdatedAt": "2026-08-22T04:00:00Z",
19219                },
19220                "attribute_types": {
19221                    "Attempt": "int",
19222                    "Escalated": "bool",
19223                    "OrderStatus": "keyword",
19224                    "Regions": "keyword_list",
19225                    "UpdatedAt": "datetime",
19226                },
19227            })]
19228        );
19229
19230        let replay = workflow_context(vec![history_event(
19231            "SearchAttributesUpserted",
19232            json!({
19233                "sequence": 6,
19234                "attributes": {
19235                    "Attempt": 3,
19236                    "Escalated": false,
19237                    "LegacyStatus": null,
19238                    "OrderStatus": "waiting",
19239                    "Regions": ["us-east", "eu-west"],
19240                    "UpdatedAt": "2026-08-22T04:00:00Z",
19241                },
19242                "attribute_types": {
19243                    "Attempt": "int",
19244                    "Escalated": "bool",
19245                    "OrderStatus": "keyword",
19246                    "Regions": "keyword_list",
19247                    "UpdatedAt": "datetime",
19248                },
19249                "merged": {},
19250            }),
19251        )]);
19252        replay
19253            .upsert_search_attributes(update)
19254            .expect("matching update replays");
19255        assert!(replay.take_commands().expect("commands").is_empty());
19256        replay.ensure_history_consumed().expect("history consumed");
19257
19258        let type_drift = workflow_context(vec![history_event(
19259            "SearchAttributesUpserted",
19260            json!({
19261                "sequence": 7,
19262                "attributes": {"OrderStatus": "waiting"},
19263                "attribute_types": {"OrderStatus": "keyword"},
19264                "merged": {"OrderStatus": "waiting"},
19265            }),
19266        )]);
19267        let error = type_drift
19268            .upsert_search_attributes(
19269                SearchAttributeUpdate::new()
19270                    .string("OrderStatus", "waiting")
19271                    .expect("string update"),
19272            )
19273            .expect_err("same JSON value with a changed type must fail replay");
19274        assert!(matches!(
19275            error,
19276            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19277                if reason == "search_attribute_type_mismatch"
19278        ));
19279
19280        let malformed_types = WorkflowState::new(
19281            vec![history_event(
19282                "SearchAttributesUpserted",
19283                json!({
19284                    "sequence": 8,
19285                    "attributes": {"OrderStatus": "waiting"},
19286                    "attribute_types": {"OrderStatus": "unsupported"},
19287                    "merged": {"OrderStatus": "waiting"},
19288                }),
19289            )],
19290            "rust-workers".to_string(),
19291            DEFAULT_CODEC.to_string(),
19292            None,
19293        )
19294        .expect_err("unsupported search-attribute type metadata must fail");
19295        assert!(matches!(
19296            malformed_types,
19297            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19298                if reason == "search_attribute_types_malformed"
19299        ));
19300
19301        assert!(matches!(
19302            SearchAttributeUpdate::new().keyword("bad key", "value"),
19303            Err(SearchAttributeUpdateError::InvalidKey(_))
19304        ));
19305        assert!(matches!(
19306            SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19307            Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19308        ));
19309        assert!(matches!(
19310            SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19311            Err(SearchAttributeUpdateError::ValueTooLong { .. })
19312        ));
19313        assert!(matches!(
19314            SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19315            Err(SearchAttributeUpdateError::InvalidDateTime(_))
19316        ));
19317        assert!(matches!(
19318            workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19319            Err(Error::InvalidSearchAttributeUpdate(
19320                SearchAttributeUpdateError::Empty
19321            ))
19322        ));
19323    }
19324
19325    #[test]
19326    fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19327        let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19328        let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19329
19330        assert!(SearchAttributeUpdate::new()
19331            .string("AsciiDescription", ascii)
19332            .is_ok());
19333        assert!(SearchAttributeUpdate::new()
19334            .string("Utf8Description", utf8)
19335            .is_ok());
19336        assert!(matches!(
19337            SearchAttributeUpdate::new().string(
19338                "TooLongDescription",
19339                "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19340            ),
19341            Err(SearchAttributeUpdateError::ValueTooLong {
19342                kind: "string",
19343                limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19344                ..
19345            })
19346        ));
19347    }
19348
19349    #[test]
19350    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19351        let lone_fire = WorkflowState::new(
19352            vec![history_event(
19353                "TimerFired",
19354                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19355            )],
19356            "rust-workers".to_string(),
19357            DEFAULT_CODEC.to_string(),
19358            None,
19359        )
19360        .expect_err("TimerFired requires TimerScheduled");
19361        assert!(matches!(
19362            lone_fire,
19363            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19364                if reason == "timer_schedule_missing_or_duplicate"
19365        ));
19366
19367        let wrong_identity = WorkflowState::new(
19368            vec![
19369                history_event(
19370                    "TimerScheduled",
19371                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19372                ),
19373                history_event(
19374                    "TimerFired",
19375                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19376                ),
19377            ],
19378            "rust-workers".to_string(),
19379            DEFAULT_CODEC.to_string(),
19380            None,
19381        )
19382        .expect_err("fire must match scheduled timer identity");
19383        assert!(matches!(
19384            wrong_identity,
19385            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19386                if reason == "timer_identity_mismatch"
19387        ));
19388
19389        let duplicate_fire = WorkflowState::new(
19390            vec![
19391                history_event(
19392                    "TimerScheduled",
19393                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19394                ),
19395                history_event(
19396                    "TimerFired",
19397                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19398                ),
19399                history_event(
19400                    "TimerFired",
19401                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19402                ),
19403            ],
19404            "rust-workers".to_string(),
19405            DEFAULT_CODEC.to_string(),
19406            None,
19407        )
19408        .expect_err("a durable timer cannot fire twice");
19409        assert!(matches!(
19410            duplicate_fire,
19411            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19412                if reason == "duplicate_timer_fire"
19413        ));
19414
19415        let wrong_fired_delay = WorkflowState::new(
19416            vec![
19417                history_event(
19418                    "TimerScheduled",
19419                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19420                ),
19421                history_event(
19422                    "TimerFired",
19423                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19424                ),
19425            ],
19426            "rust-workers".to_string(),
19427            DEFAULT_CODEC.to_string(),
19428            None,
19429        )
19430        .expect_err("timer schedule and fire delays must agree");
19431        assert!(matches!(
19432            wrong_fired_delay,
19433            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19434                if reason == "timer_history_delay_mismatch"
19435        ));
19436    }
19437
19438    #[test]
19439    fn replay_rejects_activity_moved_before_recorded_timer() {
19440        let ctx = workflow_context(vec![
19441            history_event(
19442                "TimerScheduled",
19443                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19444            ),
19445            history_event(
19446                "TimerFired",
19447                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19448            ),
19449            history_event(
19450                "ActivityCompleted",
19451                json!({
19452                    "sequence": 2,
19453                    "activity_type": "after-timer",
19454                    "payload_codec": DEFAULT_CODEC,
19455                    "result": fixture_envelope(json!("done")),
19456                }),
19457            ),
19458        ]);
19459        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19460        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19461
19462        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19463            activity.as_mut().poll(&mut task_context)
19464        else {
19465            panic!("reordered durable command must be rejected");
19466        };
19467        assert_eq!(failure.reason, "recorded_command_mismatch");
19468        assert_eq!(failure.sequence, Some(1));
19469        assert_eq!(failure.expected.as_deref(), Some("timer"));
19470        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19471    }
19472
19473    #[test]
19474    fn workflow_context_emits_a_typed_named_signal_wait() {
19475        let ctx = workflow_context(Vec::new());
19476        let mut signal = Box::pin(ctx.wait_signal("finish"));
19477        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19478
19479        assert!(matches!(
19480            signal.as_mut().poll(&mut task_context),
19481            Poll::Pending
19482        ));
19483        assert_eq!(
19484            ctx.take_commands().expect("signal-wait command"),
19485            vec![json!({
19486                "type": "open_signal_wait",
19487                "signal_name": "finish",
19488            })]
19489        );
19490    }
19491
19492    #[test]
19493    fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19494        let ctx = workflow_context(Vec::new());
19495        let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19496        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19497
19498        let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19499        else {
19500            panic!("runtime-reserved signal should be rejected");
19501        };
19502        assert!(message.contains("reserved by the workflow runtime"));
19503        assert!(ctx.take_commands().expect("commands").is_empty());
19504    }
19505
19506    #[tokio::test]
19507    async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19508        let client = Client::builder("http://127.0.0.1:9")
19509            .build()
19510            .expect("client");
19511        let error = client
19512            .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19513            .await
19514            .expect_err("runtime-reserved signal should be rejected before transport");
19515
19516        assert!(
19517            matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19518        );
19519    }
19520
19521    #[test]
19522    fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19523        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19524            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19525                value.to_string(),
19526            )]))
19527            .expect("message payload");
19528            json!({
19529                "schema": MESSAGE_STREAM_SCHEMA,
19530                "stream_name": "orders",
19531                "message_id": message_id,
19532                "position": position,
19533                "payload_envelope": payload,
19534            })
19535        }
19536
19537        fn opened(sequence: u64) -> HistoryEvent {
19538            history_event(
19539                "SignalWaitOpened",
19540                json!({
19541                    "sequence": sequence,
19542                    "signal_name": MESSAGE_STREAM_SIGNAL,
19543                }),
19544            )
19545        }
19546
19547        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19548            history_event(
19549                "SignalApplied",
19550                json!({
19551                    "sequence": sequence,
19552                    "signal_name": MESSAGE_STREAM_SIGNAL,
19553                    "value": fixture_envelope(json!([delivery])),
19554                }),
19555            )
19556        }
19557
19558        fn received(delivery: Value) -> HistoryEvent {
19559            history_event(
19560                "SignalReceived",
19561                json!({
19562                    "signal_name": MESSAGE_STREAM_SIGNAL,
19563                    "arguments": fixture_envelope(json!([delivery])),
19564                    "payload_codec": DEFAULT_CODEC,
19565                }),
19566            )
19567        }
19568
19569        let client = Client::new("http://127.0.0.1:8080").expect("client");
19570        let mut worker = Worker::new(client, "rust-workers");
19571        worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19572            let messages = ctx.message_stream("orders")?.receive(2).await?;
19573            Ok(json!(messages
19574                .into_iter()
19575                .map(|message| message.message_id)
19576                .collect::<Vec<_>>()))
19577        });
19578
19579        let first = delivery("message-1", 1, "one");
19580        let second = delivery("message-2", 2, "two");
19581        let batch = worker
19582            .execute_workflow_task_decision(workflow_task(
19583                "rust.message-stream-batch",
19584                vec![
19585                    opened(1),
19586                    received(first.clone()),
19587                    applied(1, first.clone()),
19588                    received(first.clone()),
19589                    received(second),
19590                ],
19591                DEFAULT_CODEC,
19592            ))
19593            .expect("worker task consumes the available batch");
19594
19595        assert_eq!(batch.commands.len(), 1);
19596        assert_eq!(batch.commands[0]["type"], "complete_workflow");
19597        assert_eq!(
19598            decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19599                .expect("workflow result"),
19600            json!(["message-1", "message-2"])
19601        );
19602        assert_eq!(
19603            batch.message_stream_cursors,
19604            vec![json!({"stream_name": "orders", "through_position": 2})]
19605        );
19606        assert!(batch.message_stream_waits.is_empty());
19607
19608        let partial = worker
19609            .execute_workflow_task_decision(workflow_task(
19610                "rust.message-stream-batch",
19611                vec![opened(1), received(first.clone()), applied(1, first)],
19612                DEFAULT_CODEC,
19613            ))
19614            .expect("worker task returns without waiting for a missing second item");
19615        assert_eq!(partial.commands.len(), 1);
19616        assert_eq!(partial.commands[0]["type"], "complete_workflow");
19617        assert_eq!(
19618            decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19619                .expect("workflow result"),
19620            json!(["message-1"])
19621        );
19622        assert_eq!(
19623            partial.message_stream_cursors,
19624            vec![json!({"stream_name": "orders", "through_position": 1})]
19625        );
19626        assert!(partial.message_stream_waits.is_empty());
19627    }
19628
19629    #[test]
19630    fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19631        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19632            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19633                value.to_string(),
19634            )]))
19635            .expect("message payload");
19636            json!({
19637                "schema": MESSAGE_STREAM_SCHEMA,
19638                "stream_name": "orders",
19639                "message_id": message_id,
19640                "position": position,
19641                "payload_envelope": payload,
19642            })
19643        }
19644
19645        fn opened(sequence: u64) -> HistoryEvent {
19646            history_event(
19647                "SignalWaitOpened",
19648                json!({
19649                    "sequence": sequence,
19650                    "signal_name": MESSAGE_STREAM_SIGNAL,
19651                }),
19652            )
19653        }
19654
19655        fn received(delivery: Value) -> HistoryEvent {
19656            history_event(
19657                "SignalReceived",
19658                json!({
19659                    "signal_name": MESSAGE_STREAM_SIGNAL,
19660                    "arguments": fixture_envelope(json!([delivery])),
19661                    "payload_codec": DEFAULT_CODEC,
19662                }),
19663            )
19664        }
19665
19666        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19667            history_event(
19668                "SignalApplied",
19669                json!({
19670                    "sequence": sequence,
19671                    "signal_name": MESSAGE_STREAM_SIGNAL,
19672                    "value": fixture_envelope(json!([delivery])),
19673                }),
19674            )
19675        }
19676
19677        let client = Client::new("http://127.0.0.1:8080").expect("client");
19678        let mut worker = Worker::new(client, "rust-workers");
19679        worker.register_workflow(
19680            "rust.message-stream-partial-batches",
19681            |ctx, _input| async move {
19682                let stream = ctx.message_stream("orders")?;
19683                let first = stream.receive(10).await?;
19684                let second = stream.receive(10).await?;
19685                Ok(json!([
19686                    first
19687                        .into_iter()
19688                        .map(|message| message.message_id)
19689                        .collect::<Vec<_>>(),
19690                    second
19691                        .into_iter()
19692                        .map(|message| message.message_id)
19693                        .collect::<Vec<_>>(),
19694                ]))
19695            },
19696        );
19697
19698        let first = delivery("message-1", 1, "one");
19699        let second = delivery("message-2", 2, "two");
19700        let decision = worker
19701            .execute_workflow_task_decision(workflow_task(
19702                "rust.message-stream-partial-batches",
19703                vec![
19704                    opened(1),
19705                    received(first.clone()),
19706                    applied(1, first),
19707                    opened(2),
19708                    received(second.clone()),
19709                    applied(2, second),
19710                ],
19711                DEFAULT_CODEC,
19712            ))
19713            .expect("cold replay preserves both authored receive boundaries");
19714
19715        assert_eq!(decision.commands.len(), 1);
19716        assert_eq!(decision.commands[0]["type"], "complete_workflow");
19717        assert_eq!(
19718            decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19719                .expect("workflow result"),
19720            json!([["message-1"], ["message-2"]])
19721        );
19722        assert_eq!(
19723            decision.message_stream_cursors,
19724            vec![json!({"stream_name": "orders", "through_position": 2})]
19725        );
19726        assert!(decision.message_stream_waits.is_empty());
19727    }
19728
19729    #[test]
19730    fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19731        let ctx = workflow_context(Vec::new());
19732        let stream = ctx.message_stream("orders").expect("message stream");
19733        let mut receive = Box::pin(stream.receive(10));
19734        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19735
19736        assert!(matches!(
19737            receive.as_mut().poll(&mut task_context),
19738            Poll::Pending
19739        ));
19740        assert_eq!(
19741            ctx.take_commands().expect("message-stream wait command"),
19742            vec![json!({
19743                "type": "open_signal_wait",
19744                "signal_name": MESSAGE_STREAM_SIGNAL,
19745            })]
19746        );
19747        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19748        assert!(cursors.is_empty());
19749        assert_eq!(
19750            waits,
19751            vec![json!({"stream_name": "orders", "after_position": 0})]
19752        );
19753    }
19754
19755    #[test]
19756    fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
19757        let ctx = workflow_context(vec![history_event(
19758            "SignalReceived",
19759            json!({
19760                "signal_name": MESSAGE_STREAM_SIGNAL,
19761                "arguments": fixture_envelope(json!([{
19762                    "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
19763                    "stream_name": "orders",
19764                    "through_position": 2,
19765                }])),
19766                "payload_codec": DEFAULT_CODEC,
19767            }),
19768        )]);
19769        let stream = ctx.message_stream("orders").expect("message stream");
19770        let mut receive = Box::pin(stream.receive(10));
19771        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19772
19773        assert!(matches!(
19774            receive.as_mut().poll(&mut task_context),
19775            Poll::Pending
19776        ));
19777        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19778        assert_eq!(
19779            cursors,
19780            vec![json!({"stream_name": "orders", "through_position": 2})]
19781        );
19782        assert_eq!(
19783            waits,
19784            vec![json!({"stream_name": "orders", "after_position": 2})]
19785        );
19786    }
19787
19788    #[test]
19789    fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
19790        let mut empty_map = BTreeMap::new();
19791        let mut nested = BTreeMap::new();
19792        nested.insert(
19793            "value".to_string(),
19794            AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
19795        );
19796        let values = vec![
19797            AvroValue::Bytes(vec![0, 255]),
19798            AvroValue::Long(1),
19799            AvroValue::Double(1.0),
19800            AvroValue::Array(Vec::new()),
19801            AvroValue::Map(std::mem::take(&mut empty_map)),
19802            AvroValue::Map(nested),
19803        ];
19804        let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
19805        let transport = vec![json!({
19806            "schema": MESSAGE_STREAM_SCHEMA,
19807            "stream_name": "orders",
19808            "message_id": "message-1",
19809            "position": 1,
19810            "payload_envelope": payload,
19811        })];
19812
19813        for _ in 0..2 {
19814            let Some(MessageStreamDelivery::Message(message)) =
19815                decode_message_stream_delivery(transport.clone()).expect("delivery")
19816            else {
19817                panic!("message delivery expected");
19818            };
19819            assert_eq!(message.arguments, values);
19820            assert!(matches!(message.arguments[1], AvroValue::Long(1)));
19821            assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
19822        }
19823    }
19824
19825    #[test]
19826    fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
19827        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19828            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19829                value.to_string(),
19830            )]))
19831            .expect("message payload");
19832            json!({
19833                "schema": MESSAGE_STREAM_SCHEMA,
19834                "stream_name": "orders",
19835                "message_id": message_id,
19836                "position": position,
19837                "payload_envelope": payload,
19838            })
19839        }
19840
19841        fn opened(sequence: u64) -> HistoryEvent {
19842            history_event(
19843                "SignalWaitOpened",
19844                json!({
19845                    "sequence": sequence,
19846                    "signal_name": MESSAGE_STREAM_SIGNAL,
19847                }),
19848            )
19849        }
19850
19851        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19852            history_event(
19853                "SignalApplied",
19854                json!({
19855                    "sequence": sequence,
19856                    "signal_name": MESSAGE_STREAM_SIGNAL,
19857                    "value": fixture_envelope(json!([delivery])),
19858                }),
19859            )
19860        }
19861
19862        fn worker() -> Worker {
19863            let client = Client::new("http://127.0.0.1:8080").expect("client");
19864            let mut worker = Worker::new(client, "rust-workers");
19865            worker.register_workflow("rust.message-stream", |ctx, _input| async move {
19866                let stream = ctx.message_stream("orders")?;
19867                let first = stream.receive_one().await?;
19868                let second = stream.receive_one().await?;
19869                Ok(json!([first.message_id, second.message_id]))
19870            });
19871            worker
19872        }
19873
19874        fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
19875            let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
19876            task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
19877            task.signal_arguments = Some(fixture_envelope(json!([delivery])));
19878            task
19879        }
19880
19881        let waiting = worker()
19882            .execute_workflow_task_decision(workflow_task(
19883                "rust.message-stream",
19884                Vec::new(),
19885                DEFAULT_CODEC,
19886            ))
19887            .expect("first worker opens the stream wait");
19888        assert_eq!(
19889            waiting.commands,
19890            vec![json!({
19891                "type": "open_signal_wait",
19892                "signal_name": MESSAGE_STREAM_SIGNAL,
19893            })]
19894        );
19895        assert!(waiting.message_stream_cursors.is_empty());
19896        assert_eq!(
19897            waiting.message_stream_waits,
19898            vec![json!({"stream_name": "orders", "after_position": 0})]
19899        );
19900
19901        let first_delivery = delivery("message-1", 1, "one");
19902        let first_arrival = worker()
19903            .execute_workflow_task_decision(task_with_resume(
19904                vec![opened(1)],
19905                first_delivery.clone(),
19906            ))
19907            .expect("replacement worker consumes the first arrival");
19908        assert_eq!(
19909            first_arrival.commands,
19910            vec![json!({
19911                "type": "open_signal_wait",
19912                "signal_name": MESSAGE_STREAM_SIGNAL,
19913            })]
19914        );
19915        assert_eq!(
19916            first_arrival.message_stream_cursors,
19917            vec![json!({"stream_name": "orders", "through_position": 1})]
19918        );
19919        assert_eq!(
19920            first_arrival.message_stream_waits,
19921            vec![json!({"stream_name": "orders", "after_position": 1})]
19922        );
19923
19924        let second_delivery = delivery("message-2", 2, "two");
19925        let first_applied = applied(1, first_delivery);
19926        let completed = worker()
19927            .execute_workflow_task_decision(task_with_resume(
19928                vec![opened(1), first_applied.clone(), opened(2)],
19929                second_delivery.clone(),
19930            ))
19931            .expect("next replacement worker consumes the second arrival");
19932        assert_eq!(completed.commands.len(), 1);
19933        assert_eq!(completed.commands[0]["type"], "complete_workflow");
19934        assert_eq!(
19935            decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
19936                .expect("workflow result"),
19937            json!(["message-1", "message-2"])
19938        );
19939        assert_eq!(
19940            completed.message_stream_cursors,
19941            vec![json!({"stream_name": "orders", "through_position": 2})]
19942        );
19943        assert!(completed.message_stream_waits.is_empty());
19944
19945        let replay_history = vec![
19946            opened(1),
19947            first_applied,
19948            opened(2),
19949            applied(2, second_delivery),
19950        ];
19951        for _cold_worker_or_restart in 0..2 {
19952            let replayed = worker()
19953                .execute_workflow_task_decision(workflow_task(
19954                    "rust.message-stream",
19955                    replay_history.clone(),
19956                    DEFAULT_CODEC,
19957                ))
19958                .expect("cold worker replays each logical message exactly once");
19959            assert_eq!(replayed.commands.len(), 1);
19960            assert_eq!(
19961                decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
19962                    .expect("replayed workflow result"),
19963                json!(["message-1", "message-2"])
19964            );
19965            assert_eq!(
19966                replayed.message_stream_cursors,
19967                vec![json!({"stream_name": "orders", "through_position": 2})]
19968            );
19969            assert!(replayed.message_stream_waits.is_empty());
19970        }
19971    }
19972
19973    #[test]
19974    fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
19975        assert!(!worker_protocol_supports_message_streams("1.14"));
19976        assert!(worker_protocol_supports_message_streams("1.15"));
19977        assert!(worker_protocol_supports_message_streams("1.16"));
19978        assert!(worker_protocol_supports_message_streams(
19979            WORKER_PROTOCOL_VERSION
19980        ));
19981        assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
19982    }
19983
19984    #[test]
19985    fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
19986        let ctx = workflow_context(vec![
19987            history_event(
19988                "ConditionWaitOpened",
19989                json!({
19990                    "sequence": 1,
19991                    "condition_wait_id": "condition:1",
19992                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19993                    "condition_key": "signal:finish",
19994                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
19995                }),
19996            ),
19997            history_event(
19998                "ConditionWaitSatisfied",
19999                json!({
20000                    "sequence": 1,
20001                    "condition_wait_id": "condition:1",
20002                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20003                    "condition_key": "signal:finish",
20004                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20005                }),
20006            ),
20007            history_event(
20008                "SignalReceived",
20009                json!({"signal_name": "finish", "arguments": []}),
20010            ),
20011        ]);
20012        let mut signal = Box::pin(ctx.wait_signal("finish"));
20013        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20014
20015        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20016            signal.as_mut().poll(&mut task_context)
20017        else {
20018            panic!("condition history must not resolve as a typed signal wait");
20019        };
20020        assert_eq!(failure.reason, "recorded_command_mismatch");
20021        assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20022    }
20023
20024    #[test]
20025    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20026        let signal_then_timer = vec![
20027            history_event(
20028                "SignalWaitOpened",
20029                json!({"sequence": 1, "signal_name": "go"}),
20030            ),
20031            history_event(
20032                "SignalApplied",
20033                json!({
20034                    "sequence": 1,
20035                    "signal_name": "go",
20036                    "value": fixture_envelope(json!(["now"])),
20037                }),
20038            ),
20039            history_event(
20040                "TimerScheduled",
20041                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20042            ),
20043            history_event(
20044                "TimerFired",
20045                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20046            ),
20047        ];
20048
20049        let ctx = workflow_context(signal_then_timer.clone());
20050        let mut signal = Box::pin(ctx.wait_signal("go"));
20051        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20052        assert!(matches!(
20053            signal.as_mut().poll(&mut task_context),
20054            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20055        ));
20056        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20057        assert!(matches!(
20058            timer.as_mut().poll(&mut task_context),
20059            Poll::Ready(Ok(()))
20060        ));
20061        ctx.ensure_history_consumed()
20062            .expect("signal and timer history consumed in order");
20063
20064        let reordered = workflow_context(signal_then_timer);
20065        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20066        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20067            timer_first.as_mut().poll(&mut task_context)
20068        else {
20069            panic!("timer cannot consume signal-wait-first history");
20070        };
20071        assert_eq!(failure.reason, "recorded_command_mismatch");
20072        assert_eq!(failure.sequence, Some(1));
20073        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20074
20075        let timer_then_signal = vec![
20076            history_event(
20077                "TimerScheduled",
20078                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20079            ),
20080            history_event(
20081                "TimerFired",
20082                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20083            ),
20084            history_event(
20085                "SignalWaitOpened",
20086                json!({"sequence": 2, "signal_name": "go"}),
20087            ),
20088            history_event(
20089                "SignalApplied",
20090                json!({
20091                    "sequence": 2,
20092                    "signal_name": "go",
20093                    "value": fixture_envelope(json!([])),
20094                }),
20095            ),
20096        ];
20097        let reordered = workflow_context(timer_then_signal);
20098        let mut signal_first = Box::pin(reordered.wait_signal("go"));
20099        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20100            signal_first.as_mut().poll(&mut task_context)
20101        else {
20102            panic!("signal wait cannot consume timer-first history");
20103        };
20104        assert_eq!(failure.reason, "recorded_command_mismatch");
20105        assert_eq!(failure.sequence, Some(1));
20106        assert_eq!(failure.expected.as_deref(), Some("timer"));
20107    }
20108
20109    #[test]
20110    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20111        let duplicate_timer = WorkflowState::new(
20112            vec![
20113                history_event(
20114                    "TimerScheduled",
20115                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20116                ),
20117                history_event(
20118                    "TimerScheduled",
20119                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20120                ),
20121            ],
20122            "rust-workers".to_string(),
20123            DEFAULT_CODEC.to_string(),
20124            None,
20125        )
20126        .expect_err("one workflow sequence cannot schedule two timers");
20127        assert!(matches!(
20128            duplicate_timer,
20129            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20130                if reason == "timer_schedule_missing_or_duplicate"
20131        ));
20132
20133        let colliding_kinds = WorkflowState::new(
20134            vec![
20135                history_event(
20136                    "TimerScheduled",
20137                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20138                ),
20139                history_event(
20140                    "ActivityCompleted",
20141                    json!({"sequence": 1, "activity_type": "same-sequence"}),
20142                ),
20143            ],
20144            "rust-workers".to_string(),
20145            DEFAULT_CODEC.to_string(),
20146            None,
20147        )
20148        .expect_err("one workflow sequence cannot identify two command kinds");
20149        assert!(matches!(
20150            colliding_kinds,
20151            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20152                if reason == "durable_command_sequence_collision"
20153        ));
20154
20155        let duplicate_signal_wait = WorkflowState::new(
20156            vec![
20157                history_event(
20158                    "SignalWaitOpened",
20159                    json!({"sequence": 1, "signal_name": "go"}),
20160                ),
20161                history_event(
20162                    "SignalWaitOpened",
20163                    json!({"sequence": 1, "signal_name": "go"}),
20164                ),
20165            ],
20166            "rust-workers".to_string(),
20167            DEFAULT_CODEC.to_string(),
20168            None,
20169        )
20170        .expect_err("one workflow sequence cannot open two signal waits");
20171        assert!(matches!(
20172            duplicate_signal_wait,
20173            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20174                if reason == "signal_wait_open_missing_or_duplicate"
20175        ));
20176    }
20177
20178    #[test]
20179    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20180        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20181            .expect("side-effect result");
20182        let ctx = workflow_context(vec![history_event(
20183            "SideEffectRecorded",
20184            json!({"sequence": 99, "result": result}),
20185        )]);
20186
20187        let replayed: Value = ctx
20188            .side_effect(|| panic!("recorded side effect must not run"))
20189            .expect("positive global workflow sequence is valid");
20190        assert_eq!(replayed, json!({"captured": true}));
20191        ctx.ensure_history_consumed().expect("history consumed");
20192    }
20193
20194    #[test]
20195    fn workflow_history_rejects_zero_and_descending_command_sequences() {
20196        let result =
20197            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20198        let zero = WorkflowState::new(
20199            vec![history_event(
20200                "SideEffectRecorded",
20201                json!({"sequence": 0, "result": result.clone()}),
20202            )],
20203            "rust-workers".to_string(),
20204            DEFAULT_CODEC.to_string(),
20205            None,
20206        )
20207        .expect_err("durable command sequences must be positive");
20208        assert!(matches!(
20209            zero,
20210            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20211                if reason == "durable_command_sequence_invalid"
20212        ));
20213
20214        let descending = WorkflowState::new(
20215            vec![
20216                history_event(
20217                    "SideEffectRecorded",
20218                    json!({"sequence": 3, "result": result}),
20219                ),
20220                history_event(
20221                    "VersionMarkerRecorded",
20222                    json!({
20223                        "sequence": 2,
20224                        "change_id": "descending-marker",
20225                        "version": 1,
20226                        "min_supported": 1,
20227                        "max_supported": 1,
20228                    }),
20229                ),
20230            ],
20231            "rust-workers".to_string(),
20232            DEFAULT_CODEC.to_string(),
20233            None,
20234        )
20235        .expect_err("new durable commands must remain strictly ordered");
20236        let Error::NonDeterministicReplay(failure) = descending else {
20237            panic!("expected typed replay failure");
20238        };
20239        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20240        assert_eq!(failure.sequence, Some(2));
20241        assert_eq!(
20242            failure.expected.as_deref(),
20243            Some("workflow sequence greater than 3")
20244        );
20245        assert_eq!(failure.actual.as_deref(), Some("2"));
20246    }
20247
20248    #[test]
20249    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20250        fn worker() -> Worker {
20251            let client = Client::new("http://127.0.0.1:8080").expect("client");
20252            let mut worker = Worker::new(client, "rust-workers");
20253            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20254                ctx.wait_signal("finish").await?;
20255                let marker: String =
20256                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20257                assert_eq!(marker, "after-finish");
20258                Ok(json!("finished"))
20259            });
20260            worker
20261        }
20262
20263        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20264            .expect("side-effect result");
20265        let task = workflow_task(
20266            "rust.finish-after-gaps",
20267            vec![
20268                history_event(
20269                    "SignalWaitOpened",
20270                    json!({"sequence": 1, "signal_name": "finish"}),
20271                ),
20272                history_event(
20273                    "SignalReceived",
20274                    json!({
20275                        "signal_id": "increment-3",
20276                        "signal_name": "increment",
20277                        "workflow_sequence": 2,
20278                        "payload_codec": DEFAULT_CODEC,
20279                        "arguments": fixture_envelope(json!([3])),
20280                    }),
20281                ),
20282                history_event(
20283                    "SignalReceived",
20284                    json!({
20285                        "signal_id": "increment-5",
20286                        "signal_name": "increment",
20287                        "workflow_sequence": 3,
20288                        "payload_codec": DEFAULT_CODEC,
20289                        "arguments": fixture_envelope(json!([5])),
20290                    }),
20291                ),
20292                history_event(
20293                    "SignalReceived",
20294                    json!({
20295                        "signal_id": "finish",
20296                        "signal_name": "finish",
20297                        "workflow_sequence": 4,
20298                        "payload_codec": DEFAULT_CODEC,
20299                        "arguments": fixture_envelope(json!([])),
20300                    }),
20301                ),
20302                history_event(
20303                    "SignalApplied",
20304                    json!({
20305                        "sequence": 1,
20306                        "signal_id": "finish",
20307                        "signal_name": "finish",
20308                        "payload_codec": DEFAULT_CODEC,
20309                        "value": fixture_envelope(json!([])),
20310                    }),
20311                ),
20312                history_event(
20313                    "SideEffectRecorded",
20314                    json!({"sequence": 5, "result": marker}),
20315                ),
20316            ],
20317            DEFAULT_CODEC,
20318        );
20319
20320        for _original_or_cold_worker in 0..2 {
20321            let commands = worker()
20322                .execute_workflow_task(task.clone())
20323                .expect("signal gaps preserve deterministic replay");
20324            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20325            assert_eq!(commands[0]["type"], "complete_workflow");
20326            assert_eq!(
20327                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20328                json!("finished")
20329            );
20330        }
20331    }
20332
20333    #[test]
20334    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20335        let ctx = workflow_context(Vec::new());
20336        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20337        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20338        assert!(matches!(
20339            sleep.as_mut().poll(&mut task_context),
20340            Poll::Ready(Err(Error::TimerDurationOverflow))
20341        ));
20342        assert!(ctx.take_commands().expect("commands").is_empty());
20343    }
20344
20345    #[test]
20346    fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20347        let entries = AvroValue::Map(BTreeMap::from([
20348            ("text".to_string(), AvroValue::String("same".to_string())),
20349            (
20350                "nested".to_string(),
20351                AvroValue::Map(BTreeMap::from([
20352                    ("beta".to_string(), AvroValue::Long(2)),
20353                    ("alpha".to_string(), AvroValue::Long(1)),
20354                ])),
20355            ),
20356            ("long".to_string(), AvroValue::Long(7)),
20357            ("double".to_string(), AvroValue::Double(7.0)),
20358            ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20359        ]));
20360        let ctx = workflow_context(Vec::new());
20361        ctx.upsert_memo(entries.clone()).expect("valid memo update");
20362        let commands = ctx.take_commands().expect("commands");
20363
20364        assert_eq!(commands.len(), 1);
20365        assert_eq!(commands[0]["type"], "upsert_memo");
20366        let server_entries = json!({
20367            "codec": "avro",
20368            "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20369        });
20370        assert_eq!(
20371            commands[0]["entries"]
20372                .as_object()
20373                .expect("entries envelope")
20374                .keys()
20375                .collect::<Vec<_>>(),
20376            vec!["blob", "codec"]
20377        );
20378        assert_eq!(commands[0]["entries"], server_entries);
20379        let wire_entries =
20380            decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20381        assert_eq!(wire_entries, entries);
20382
20383        let history = vec![history_event(
20384            "MemoUpserted",
20385            json!({
20386                "sequence": 1,
20387                "entries": server_entries.clone(),
20388                "merged": server_entries,
20389            }),
20390        )];
20391        let replay = workflow_context(history.clone());
20392        replay
20393            .upsert_memo(entries.clone())
20394            .expect("matching replay identity");
20395        assert!(replay.take_commands().expect("replay commands").is_empty());
20396
20397        let changed_types = AvroValue::Map(BTreeMap::from([
20398            ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20399            (
20400                "nested".to_string(),
20401                AvroValue::Map(BTreeMap::from([
20402                    ("alpha".to_string(), AvroValue::Long(1)),
20403                    ("beta".to_string(), AvroValue::Long(2)),
20404                ])),
20405            ),
20406            ("long".to_string(), AvroValue::Double(7.0)),
20407            ("double".to_string(), AvroValue::Long(7)),
20408            ("binary".to_string(), AvroValue::String("same".to_string())),
20409        ]));
20410        let error = workflow_context(history)
20411            .upsert_memo(changed_types)
20412            .expect_err("memo replay identity must preserve Avro value types");
20413        assert!(matches!(
20414            error,
20415            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20416        ));
20417    }
20418
20419    #[test]
20420    fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20421        let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20422            .expect("memo envelope");
20423        let replay = workflow_context(vec![history_event(
20424            "MemoUpserted",
20425            json!({
20426                "sequence": 1,
20427                "entries": original.clone(),
20428                "merged": original
20429            }),
20430        )]);
20431        let error = replay
20432            .upsert_memo(json!({"stage": "changed"}))
20433            .expect_err("changed memo update must fail replay");
20434        assert!(matches!(
20435            error,
20436            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20437        ));
20438
20439        let invalid = workflow_context(Vec::new())
20440            .upsert_memo(
20441                json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20442            )
20443            .expect_err("oversized key");
20444        assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20445    }
20446
20447    #[test]
20448    fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20449        let negative_zero = AvroValue::Map(BTreeMap::from([(
20450            "reading".to_string(),
20451            AvroValue::Double(-0.0),
20452        )]));
20453        let negative_zero_envelope =
20454            encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20455        let history = vec![history_event(
20456            "MemoUpserted",
20457            json!({
20458                "sequence": 1,
20459                "entries": negative_zero_envelope.clone(),
20460                "merged": negative_zero_envelope,
20461            }),
20462        )];
20463
20464        workflow_context(history.clone())
20465            .upsert_memo(negative_zero)
20466            .expect("matching negative-zero history identity");
20467
20468        let error = workflow_context(history)
20469            .upsert_memo(AvroValue::Map(BTreeMap::from([(
20470                "reading".to_string(),
20471                AvroValue::Double(0.0),
20472            )])))
20473            .expect_err("positive zero must not consume negative-zero memo history");
20474        assert!(matches!(
20475            error,
20476            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20477        ));
20478    }
20479
20480    #[test]
20481    fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20482        let supported = json!({
20483            "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20484            "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20485        });
20486        assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20487        assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20488            "workflow_memo_updates": {"supported": false},
20489            "supported_workflow_task_commands": ["upsert_memo"]
20490        }))));
20491        assert!(commands_use_workflow_memo_updates(&[json!({
20492            "type": "upsert_memo",
20493            "entries": {"stage": "processing"}
20494        })]));
20495    }
20496
20497    #[test]
20498    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20499        let client = Client::new("http://127.0.0.1:8080").expect("client");
20500        let mut worker = Worker::new(client, "rust-workers");
20501        worker.register_workflow("rust.timer", |ctx, _input| async move {
20502            ctx.sleep(Duration::from_secs(5)).await?;
20503            ctx.activity("after-timer", json!([])).await
20504        });
20505
20506        let task = |history_events| WorkflowTask {
20507            task_id: "wft-rust-timer-1".to_string(),
20508            workflow_command_id: None,
20509            workflow_id: Some("wf-rust-timer".to_string()),
20510            run_id: Some("run-rust-timer".to_string()),
20511            workflow_type: "rust.timer".to_string(),
20512            cancel_requested: false,
20513            payload_codec: DEFAULT_CODEC.to_string(),
20514            arguments: Some(
20515                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20516            ),
20517            history_events,
20518            total_history_events: None,
20519            history_size_bytes: None,
20520            continue_as_new_recommended: None,
20521            history_budget_pressure: None,
20522            next_history_page_token: None,
20523            workflow_task_attempt: 1,
20524            workflow_signal_id: None,
20525            signal_name: None,
20526            signal_arguments: None,
20527            workflow_update_id: None,
20528            update_name: None,
20529            lease_owner: Some("rust-worker".to_string()),
20530        };
20531
20532        let initial = worker
20533            .execute_workflow_task(task(Vec::new()))
20534            .expect("initial timer task");
20535        assert_eq!(
20536            initial,
20537            vec![json!({"type": "start_timer", "delay_seconds": 5})]
20538        );
20539
20540        let activity_result =
20541            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20542        let replayed = worker
20543            .execute_workflow_task(task(vec![
20544                history_event(
20545                    "TimerScheduled",
20546                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20547                ),
20548                history_event(
20549                    "TimerFired",
20550                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20551                ),
20552                history_event(
20553                    "ActivityCompleted",
20554                    json!({
20555                        "sequence": 2,
20556                        "activity_type": "after-timer",
20557                        "payload_codec": DEFAULT_CODEC,
20558                        "result": activity_result,
20559                    }),
20560                ),
20561            ]))
20562            .expect("replayed workflow task");
20563        assert_eq!(replayed.len(), 1);
20564        assert_eq!(replayed[0]["type"], "complete_workflow");
20565        assert_eq!(
20566            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20567            json!("done")
20568        );
20569    }
20570
20571    #[test]
20572    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20573        let client = Client::new("http://127.0.0.1:8080").expect("client");
20574        let mut worker = Worker::new(client, "rust-workers");
20575        worker.register_workflow("rust.continue", |ctx, _input| async move {
20576            ctx.continue_as_new_with_options(
20577                ContinueAsNewOptions::new()
20578                    .workflow_type("rust.next")
20579                    .task_queue("next-workers"),
20580                json!([2, {"cursor": "next"}]),
20581            )
20582        });
20583
20584        let commands = worker
20585            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20586            .expect("continue-as-new command");
20587
20588        assert_eq!(commands.len(), 1);
20589        assert_eq!(commands[0]["type"], "continue_as_new");
20590        assert_eq!(commands[0]["workflow_type"], "rust.next");
20591        assert_eq!(commands[0]["queue"], "next-workers");
20592        assert_eq!(
20593            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20594                .expect("continue-as-new arguments"),
20595            json!([2, {"cursor": "next"}])
20596        );
20597    }
20598
20599    #[test]
20600    fn continue_as_new_preserves_typed_arguments() {
20601        let client = Client::new("http://127.0.0.1:8080").expect("client");
20602        let mut worker = Worker::new(client, "rust-workers");
20603        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20604            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20605            unreachable!("continue-as-new returns a control-flow error")
20606        });
20607
20608        let commands = worker
20609            .execute_workflow_task(workflow_task(
20610                "rust.typed-continue",
20611                Vec::new(),
20612                DEFAULT_CODEC,
20613            ))
20614            .expect("typed continue-as-new command");
20615
20616        assert_eq!(commands[0]["type"], "continue_as_new");
20617        assert_eq!(
20618            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20619                .expect("typed continue arguments"),
20620            AvroValue::Array(vec![typed_fidelity_probe()])
20621        );
20622    }
20623
20624    #[test]
20625    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20626        let client = Client::new("http://127.0.0.1:8080").expect("client");
20627        let mut worker = Worker::new(client, "rust-workers");
20628        worker.register_workflow("rust.continue", |ctx, _input| async move {
20629            ctx.continue_as_new(json!([2]))
20630        });
20631        let task = workflow_task(
20632            "rust.continue",
20633            vec![history_event(
20634                "WorkflowContinuedAsNew",
20635                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20636            )],
20637            DEFAULT_CODEC,
20638        );
20639
20640        for _worker_restart_or_redelivery in 0..2 {
20641            let commands = worker
20642                .execute_workflow_task(task.clone())
20643                .expect("recorded transition replays");
20644            assert!(
20645                commands.is_empty(),
20646                "replay must not emit another successor"
20647            );
20648        }
20649    }
20650
20651    #[test]
20652    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20653        let ctx = workflow_context(Vec::new());
20654        let error = ctx
20655            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
20656            .expect_err("blank queue must be rejected");
20657
20658        let Error::InvalidContinueAsNewOptions(error) = error else {
20659            panic!("expected typed continue-as-new validation error");
20660        };
20661        assert_eq!(error.field, "task_queue");
20662        assert!(ctx.take_commands().expect("commands").is_empty());
20663    }
20664
20665    #[test]
20666    fn workflow_context_exposes_server_history_budget() {
20667        let client = Client::new("http://127.0.0.1:8080").expect("client");
20668        let mut worker = Worker::new(client, "rust-workers");
20669        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20670            let budget = ctx.history_budget()?;
20671            Ok(json!({
20672                "events": budget.event_count,
20673                "bytes": budget.size_bytes,
20674                "recommended": budget.continue_as_new_recommended,
20675                "pressure": budget.pressure,
20676            }))
20677        });
20678        let task: WorkflowTask = serde_json::from_value(json!({
20679            "task_id": "task-history-budget",
20680            "workflow_type": "rust.history-budget",
20681            "payload_codec": DEFAULT_CODEC,
20682            "history_events": [],
20683            "total_history_events": 480,
20684            "history_size_bytes": 1_048_576,
20685            "continue_as_new_recommended": true,
20686            "history_budget_pressure": "continue_as_new_recommended",
20687        }))
20688        .expect("published workflow task");
20689
20690        let commands = worker
20691            .execute_workflow_task(task)
20692            .expect("history-budget workflow");
20693        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20694        assert_eq!(result["events"], 480);
20695        assert_eq!(result["bytes"], 1_048_576);
20696        assert_eq!(result["recommended"], true);
20697        assert_eq!(result["pressure"], "continue_as_new_recommended");
20698    }
20699
20700    #[test]
20701    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20702        let client = Client::new("http://127.0.0.1:8080").expect("client");
20703        let mut worker = Worker::new(client, "rust-workers");
20704        worker.register_workflow("rust.failing", |_ctx, _input| async move {
20705            Err(Error::Codec("rust_conformance_failure".to_string()))
20706        });
20707        let task = WorkflowTask {
20708            task_id: "wft-rust-failing-1".to_string(),
20709            workflow_command_id: None,
20710            workflow_id: Some("wf-rust-failing".to_string()),
20711            run_id: Some("run-rust-failing".to_string()),
20712            workflow_type: "rust.failing".to_string(),
20713            cancel_requested: false,
20714            payload_codec: DEFAULT_CODEC.to_string(),
20715            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20716            history_events: Vec::new(),
20717            total_history_events: Some(0),
20718            history_size_bytes: None,
20719            continue_as_new_recommended: None,
20720            history_budget_pressure: None,
20721            next_history_page_token: None,
20722            workflow_task_attempt: 1,
20723            workflow_signal_id: None,
20724            signal_name: None,
20725            signal_arguments: None,
20726            workflow_update_id: None,
20727            update_name: None,
20728            lease_owner: Some("rust-worker".to_string()),
20729        };
20730
20731        let commands = worker
20732            .execute_workflow_task(task)
20733            .expect("handler failure becomes a workflow command");
20734
20735        assert_eq!(commands.len(), 1);
20736        assert_eq!(commands[0]["type"], "fail_workflow");
20737        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20738        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20739        assert_eq!(commands[0]["non_retryable"], false);
20740        assert_eq!(
20741            commands[0]["message"],
20742            "codec error: rust_conformance_failure"
20743        );
20744        assert_eq!(
20745            commands[0]["exception"]["message"],
20746            "codec error: rust_conformance_failure"
20747        );
20748    }
20749
20750    #[test]
20751    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
20752        let client = Client::new("http://127.0.0.1:8080").expect("client");
20753        let mut worker = Worker::new(client, "rust-workers");
20754        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
20755            let _: String = ctx.side_effect(|| "captured".to_string())?;
20756            Err(Error::WorkerLoop("application failure".to_string()))
20757        });
20758
20759        let commands = worker
20760            .execute_workflow_task(workflow_task(
20761                "rust.failing-after-side-effect",
20762                Vec::new(),
20763                DEFAULT_CODEC,
20764            ))
20765            .expect("ordinary failure remains a workflow decision");
20766
20767        assert_eq!(commands.len(), 2);
20768        assert_eq!(commands[0]["type"], "record_side_effect");
20769        assert_eq!(commands[1]["type"], "fail_workflow");
20770    }
20771
20772    #[test]
20773    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
20774        let client = Client::new("http://127.0.0.1:8080").expect("client");
20775        let mut worker = Worker::new(client, "rust-workers");
20776        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
20777            Err(Error::WorkerLoop("application failure".to_string()))
20778        });
20779        let result =
20780            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
20781
20782        let error = worker
20783            .execute_workflow_task(workflow_task(
20784                "rust.removed-side-effect",
20785                vec![history_event(
20786                    "SideEffectRecorded",
20787                    json!({"sequence": 1, "result": result}),
20788                )],
20789                DEFAULT_CODEC,
20790            ))
20791            .expect_err("removed committed history must not become fail_workflow");
20792
20793        let Error::NonDeterministicReplay(failure) = error else {
20794            panic!("expected typed replay failure");
20795        };
20796        assert_eq!(failure.reason, "recorded_commands_unconsumed");
20797        assert_eq!(failure.sequence, Some(1));
20798        assert_eq!(failure.expected.as_deref(), Some("side effect"));
20799    }
20800
20801    #[test]
20802    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
20803        let client = Client::new("http://127.0.0.1:8080").expect("client");
20804        let mut worker = Worker::new(client, "rust-workers");
20805        worker.register_workflow(
20806            "rust.side-effect-before-marker-error",
20807            |ctx, _input| async move {
20808                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
20809                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
20810                ctx.get_version("restart-safe", 2, 2)?;
20811                Ok(Value::Null)
20812            },
20813        );
20814
20815        let error = worker
20816            .execute_workflow_task(workflow_task(
20817                "rust.side-effect-before-marker-error",
20818                vec![history_event(
20819                    "VersionMarkerRecorded",
20820                    json!({
20821                        "sequence": 1,
20822                        "change_id": "restart-safe",
20823                        "version": 1,
20824                        "min_supported": 1,
20825                        "max_supported": 1,
20826                    }),
20827                )],
20828                DEFAULT_CODEC,
20829            ))
20830            .expect_err("replay error must return no queued workflow commands");
20831
20832        let Error::NonDeterministicReplay(failure) = error else {
20833            panic!("expected typed replay failure");
20834        };
20835        assert_eq!(failure.reason, "version_marker_incompatible_range");
20836        assert_eq!(failure.sequence, Some(1));
20837    }
20838
20839    #[test]
20840    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
20841        let client = Client::new("http://127.0.0.1:8080").expect("client");
20842        let mut worker = Worker::new(client, "rust-workers");
20843        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
20844            ctx.sleep(Duration::from_secs(5)).await?;
20845            Ok(json!({"status": "timer fired"}))
20846        });
20847
20848        let task = WorkflowTask {
20849            task_id: "wft-rust-timer-pending".to_string(),
20850            workflow_command_id: None,
20851            workflow_id: Some("wf-rust-timer".to_string()),
20852            run_id: Some("run-rust-timer".to_string()),
20853            workflow_type: "rust.timer.pending".to_string(),
20854            cancel_requested: false,
20855            payload_codec: DEFAULT_CODEC.to_string(),
20856            arguments: Some(
20857                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20858            ),
20859            history_events: vec![history_event(
20860                "TimerScheduled",
20861                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20862            )],
20863            total_history_events: Some(1),
20864            history_size_bytes: None,
20865            continue_as_new_recommended: None,
20866            history_budget_pressure: None,
20867            next_history_page_token: None,
20868            workflow_task_attempt: 1,
20869            workflow_signal_id: None,
20870            signal_name: None,
20871            signal_arguments: None,
20872            workflow_update_id: None,
20873            update_name: None,
20874            lease_owner: Some("rust-worker".to_string()),
20875        };
20876
20877        for _redelivery_or_restart in 0..2 {
20878            let commands = worker
20879                .execute_workflow_task(task.clone())
20880                .expect("recorded timer remains pending");
20881            assert!(
20882                commands.is_empty(),
20883                "recorded timer must not be rescheduled"
20884            );
20885        }
20886    }
20887
20888    #[test]
20889    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
20890        let client = Client::new("http://127.0.0.1:8080").expect("client");
20891        let mut worker = Worker::new(client, "rust-workers");
20892        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
20893            Ok(json!({"status": "completed"}))
20894        });
20895        let task = WorkflowTask {
20896            task_id: "wft-rust-timer-removed".to_string(),
20897            workflow_command_id: None,
20898            workflow_id: Some("wf-rust-timer".to_string()),
20899            run_id: Some("run-rust-timer".to_string()),
20900            workflow_type: "rust.timer.removed".to_string(),
20901            cancel_requested: false,
20902            payload_codec: DEFAULT_CODEC.to_string(),
20903            arguments: Some(
20904                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20905            ),
20906            history_events: vec![
20907                history_event(
20908                    "TimerScheduled",
20909                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20910                ),
20911                history_event(
20912                    "TimerFired",
20913                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20914                ),
20915            ],
20916            total_history_events: Some(2),
20917            history_size_bytes: None,
20918            continue_as_new_recommended: None,
20919            history_budget_pressure: None,
20920            next_history_page_token: None,
20921            workflow_task_attempt: 1,
20922            workflow_signal_id: None,
20923            signal_name: None,
20924            signal_arguments: None,
20925            workflow_update_id: None,
20926            update_name: None,
20927            lease_owner: Some("rust-worker".to_string()),
20928        };
20929
20930        let Error::NonDeterministicReplay(failure) = worker
20931            .execute_workflow_task(task)
20932            .expect_err("removed timer must fail replay")
20933        else {
20934            panic!("expected typed replay failure");
20935        };
20936        assert_eq!(failure.reason, "recorded_commands_unconsumed");
20937        assert_eq!(failure.sequence, Some(1));
20938    }
20939
20940    #[test]
20941    fn workflow_context_emits_explicit_child_workflow_contract() {
20942        let ctx = WorkflowContext {
20943            state: Arc::new(Mutex::new(
20944                WorkflowState::new_with_identity(
20945                    Vec::new(),
20946                    Some("wf-parent".to_string()),
20947                    Some("run-parent".to_string()),
20948                    "parent-workers".to_string(),
20949                    DEFAULT_CODEC.to_string(),
20950                    None,
20951                )
20952                .expect("workflow state"),
20953            )),
20954        };
20955        let options = ChildWorkflowOptions::new("python-workers")
20956            .parent_close_policy(ParentClosePolicy::RequestCancel)
20957            .retry_policy(ChildWorkflowRetryPolicy {
20958                max_attempts: Some(3),
20959                backoff_seconds: vec![1, 5],
20960                non_retryable_error_types: vec!["ValidationError".to_string()],
20961            })
20962            .execution_timeout_seconds(600)
20963            .run_timeout_seconds(120);
20964        let mut call = Box::pin(ctx.start_child_workflow(
20965            "python.fulfil-order",
20966            options,
20967            json!([{"order_id": "order-42"}]),
20968        ));
20969        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20970
20971        assert!(matches!(
20972            call.as_mut().poll(&mut task_context),
20973            Poll::Pending
20974        ));
20975        let commands = ctx.take_commands().expect("commands");
20976        assert_eq!(commands.len(), 1);
20977        let command = &commands[0];
20978        assert_eq!(command["type"], "start_child_workflow");
20979        assert_eq!(command["workflow_type"], "python.fulfil-order");
20980        assert_eq!(command["queue"], "python-workers");
20981        assert_eq!(command["parent_close_policy"], "request_cancel");
20982        assert_eq!(command["retry_policy"]["max_attempts"], 3);
20983        assert_eq!(command["execution_timeout_seconds"], 600);
20984        assert_eq!(command["run_timeout_seconds"], 120);
20985        assert_eq!(
20986            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
20987            json!([{"order_id": "order-42"}])
20988        );
20989    }
20990
20991    fn child_parent_worker() -> Worker {
20992        let client = Client::new("http://127.0.0.1:8080").expect("client");
20993        let mut worker = Worker::new(client, "rust-parent-workers");
20994        worker.register_workflow("rust.parent", |ctx, _input| async move {
20995            let child = ctx
20996                .start_child_workflow(
20997                    "python.child",
20998                    ChildWorkflowOptions::new("python-child-workers")
20999                        .parent_close_policy(ParentClosePolicy::Terminate),
21000                    json!([{"codec_probe": [1, true, "rust"]}]),
21001                )
21002                .await?;
21003            Ok(json!({
21004                "parent_workflow_id": child.parent.workflow_id,
21005                "parent_run_id": child.parent.run_id,
21006                "child_workflow_id": child.child.workflow_id,
21007                "child_run_id": child.child.run_id,
21008                "child_workflow_type": child.child_workflow_type,
21009                "result": child.result,
21010            }))
21011        });
21012        worker
21013    }
21014
21015    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21016        WorkflowTask {
21017            task_id: "wft-child-parent".to_string(),
21018            workflow_command_id: None,
21019            workflow_id: Some("wf-parent".to_string()),
21020            run_id: Some("run-parent".to_string()),
21021            workflow_type: "rust.parent".to_string(),
21022            cancel_requested: false,
21023            payload_codec: DEFAULT_CODEC.to_string(),
21024            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21025            history_events: vec![
21026                HistoryEvent {
21027                    event_type: "ChildWorkflowScheduled".to_string(),
21028                    payload: json!({
21029                        "sequence": 1,
21030                        "child_call_id": "call-child",
21031                        "child_workflow_instance_id": "wf-child",
21032                        "child_workflow_run_id": "run-child",
21033                        "child_workflow_type": "python.child",
21034                    }),
21035                    raw: HashMap::new(),
21036                },
21037                HistoryEvent {
21038                    event_type: event_type.to_string(),
21039                    payload,
21040                    raw: HashMap::new(),
21041                },
21042            ],
21043            total_history_events: Some(2),
21044            history_size_bytes: None,
21045            continue_as_new_recommended: None,
21046            history_budget_pressure: None,
21047            next_history_page_token: None,
21048            workflow_task_attempt: 1,
21049            workflow_signal_id: None,
21050            signal_name: None,
21051            signal_arguments: None,
21052            workflow_update_id: None,
21053            update_name: None,
21054            lease_owner: Some("rust-worker".to_string()),
21055        }
21056    }
21057
21058    #[test]
21059    fn committed_child_result_replays_without_starting_a_duplicate() {
21060        let worker = child_parent_worker();
21061        let task = child_parent_task(
21062            "ChildRunCompleted",
21063            json!({
21064                "sequence": 1,
21065                "child_call_id": "call-child",
21066                "child_workflow_instance_id": "wf-child",
21067                "child_workflow_run_id": "run-child",
21068                "child_workflow_type": "python.child",
21069                "payload_codec": DEFAULT_CODEC,
21070                "result": fixture_envelope(json!({"from":"python","ok":true})),
21071            }),
21072        );
21073
21074        for _restart in 0..2 {
21075            let commands = worker
21076                .execute_workflow_task(task.clone())
21077                .expect("replayed parent task");
21078            assert_eq!(commands.len(), 1);
21079            assert_eq!(commands[0]["type"], "complete_workflow");
21080            assert!(!commands
21081                .iter()
21082                .any(|command| command["type"] == "start_child_workflow"));
21083            let output =
21084                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21085            assert_eq!(output["parent_workflow_id"], "wf-parent");
21086            assert_eq!(output["parent_run_id"], "run-parent");
21087            assert_eq!(output["child_workflow_id"], "wf-child");
21088            assert_eq!(output["child_run_id"], "run-child");
21089            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21090        }
21091    }
21092
21093    #[test]
21094    fn typed_child_arguments_and_results_survive_replay() {
21095        let client = Client::new("http://127.0.0.1:8080").expect("client");
21096        let mut worker = Worker::new(client, "rust-parent-workers");
21097        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21098            let child = ctx
21099                .start_child_workflow_avro_value(
21100                    "python.typed-child",
21101                    ChildWorkflowOptions::new("python-workers"),
21102                    AvroValue::Array(vec![typed_fidelity_probe()]),
21103                )
21104                .await?;
21105            Ok(child.result)
21106        });
21107
21108        let initial = worker
21109            .execute_workflow_task(workflow_task(
21110                "rust.typed-parent",
21111                Vec::new(),
21112                DEFAULT_CODEC,
21113            ))
21114            .expect("typed child start");
21115        assert_eq!(initial[0]["type"], "start_child_workflow");
21116        assert_eq!(
21117            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21118                .expect("typed child arguments"),
21119            AvroValue::Array(vec![typed_fidelity_probe()])
21120        );
21121
21122        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21123            .expect("typed child result");
21124        let task = workflow_task(
21125            "rust.typed-parent",
21126            vec![
21127                history_event(
21128                    "ChildWorkflowScheduled",
21129                    json!({
21130                        "sequence": 1,
21131                        "child_call_id": "call-typed",
21132                        "child_workflow_instance_id": "wf-child",
21133                        "child_workflow_run_id": "run-child",
21134                        "child_workflow_type": "python.typed-child",
21135                    }),
21136                ),
21137                history_event(
21138                    "ChildRunCompleted",
21139                    json!({
21140                        "sequence": 1,
21141                        "child_call_id": "call-typed",
21142                        "child_workflow_instance_id": "wf-child",
21143                        "child_workflow_run_id": "run-child",
21144                        "child_workflow_type": "python.typed-child",
21145                        "payload_codec": DEFAULT_CODEC,
21146                        "result": result,
21147                    }),
21148                ),
21149            ],
21150            DEFAULT_CODEC,
21151        );
21152
21153        let commands = worker
21154            .execute_workflow_task(task)
21155            .expect("typed child replay");
21156        assert_eq!(commands[0]["type"], "complete_workflow");
21157        assert_eq!(
21158            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21159                .expect("typed parent result"),
21160            typed_fidelity_probe()
21161        );
21162    }
21163
21164    #[test]
21165    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21166        let worker = child_parent_worker();
21167        let mut task = child_parent_task("unused", Value::Null);
21168        task.history_events.truncate(1);
21169        task.total_history_events = Some(1);
21170
21171        for _redelivery_or_restart in 0..2 {
21172            let commands = worker
21173                .execute_workflow_task(task.clone())
21174                .expect("recorded child remains pending");
21175            assert!(
21176                commands.is_empty(),
21177                "recorded pending child must not be started again"
21178            );
21179        }
21180    }
21181
21182    #[test]
21183    fn child_cancellation_becomes_stable_parent_failure_command() {
21184        let worker = child_parent_worker();
21185        let task = child_parent_task(
21186            "ChildRunCancelled",
21187            json!({
21188                "sequence": 1,
21189                "child_workflow_instance_id": "wf-child",
21190                "child_workflow_run_id": "run-child",
21191                "child_workflow_type": "python.child",
21192                "failure_id": "failure-child",
21193                "failure_category": "cancelled",
21194                "message": "cancelled by parent-close policy",
21195            }),
21196        );
21197
21198        let commands = worker
21199            .execute_workflow_task(task)
21200            .expect("parent settlement");
21201        assert_eq!(commands.len(), 1);
21202        assert_eq!(commands[0]["type"], "fail_workflow");
21203        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21204        assert_eq!(
21205            commands[0]["exception"]["properties"]["reason"],
21206            "cancelled"
21207        );
21208        assert_eq!(
21209            commands[0]["exception"]["properties"]["child_workflow_run_id"],
21210            "run-child"
21211        );
21212    }
21213
21214    #[test]
21215    fn workflow_can_handle_typed_child_failure() {
21216        let client = Client::new("http://127.0.0.1:8080").expect("client");
21217        let mut worker = Worker::new(client, "rust-parent-workers");
21218        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21219            match ctx
21220                .start_child_workflow(
21221                    "python.child",
21222                    ChildWorkflowOptions::new("python-child-workers"),
21223                    json!([]),
21224                )
21225                .await
21226            {
21227                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21228                    "reason": failure.reason,
21229                    "failure_id": failure.failure_id,
21230                    "exception_class": failure.exception_class,
21231                    "child_run_id": failure.child_workflow_run_id,
21232                })),
21233                Err(error) => Err(error),
21234                Ok(_) => Err(Error::WorkerLoop(
21235                    "child unexpectedly succeeded".to_string(),
21236                )),
21237            }
21238        });
21239        let mut task = child_parent_task(
21240            "ChildRunFailed",
21241            json!({
21242                "sequence": 1,
21243                "child_workflow_instance_id": "wf-child",
21244                "child_workflow_run_id": "run-child",
21245                "child_workflow_type": "python.child",
21246                "failure_id": "failure-child",
21247                "failure_category": "child_workflow",
21248                "message": "payment rejected",
21249                "exception": {
21250                    "type": "PaymentRejected",
21251                    "class": "payments.PaymentRejected",
21252                    "message": "payment rejected"
21253                }
21254            }),
21255        );
21256        task.workflow_type = "rust.handled-parent".to_string();
21257
21258        let commands = worker.execute_workflow_task(task).expect("handled failure");
21259        assert_eq!(commands[0]["type"], "complete_workflow");
21260        let output =
21261            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21262        assert_eq!(output["reason"], "child_workflow");
21263        assert_eq!(output["failure_id"], "failure-child");
21264        assert_eq!(output["exception_class"], "payments.PaymentRejected");
21265        assert_eq!(output["child_run_id"], "run-child");
21266    }
21267
21268    #[test]
21269    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21270        let client = Client::new("http://127.0.0.1:8080").expect("client");
21271        let mut worker = Worker::new(client, "rust-workers");
21272
21273        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21274            let signal = ctx.wait_signal("start").await?;
21275            let name = signal
21276                .first()
21277                .and_then(|value| value.as_str())
21278                .unwrap_or("world");
21279            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21280            Ok(json!({
21281                "greeting": greeting,
21282                "language": "rust"
21283            }))
21284        });
21285
21286        let signal_arguments =
21287            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21288        let task = WorkflowTask {
21289            task_id: "wft-rust-signal-1".to_string(),
21290            workflow_command_id: None,
21291            workflow_id: Some("wf-rust-hello".to_string()),
21292            run_id: Some("run-rust-hello".to_string()),
21293            workflow_type: "rust.hello_workflow".to_string(),
21294            cancel_requested: false,
21295            payload_codec: DEFAULT_CODEC.to_string(),
21296            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21297            history_events: vec![HistoryEvent {
21298                event_type: "SignalReceived".to_string(),
21299                payload: json!({
21300                    "signal_id": "sig-rust-1",
21301                    "signal_name": "start"
21302                }),
21303                raw: HashMap::new(),
21304            }],
21305            total_history_events: Some(1),
21306            history_size_bytes: None,
21307            continue_as_new_recommended: None,
21308            history_budget_pressure: None,
21309            next_history_page_token: None,
21310            workflow_task_attempt: 1,
21311            workflow_signal_id: Some("sig-rust-1".to_string()),
21312            signal_name: Some("start".to_string()),
21313            signal_arguments: Some(signal_arguments),
21314            workflow_update_id: None,
21315            update_name: None,
21316            lease_owner: Some("rust-worker".to_string()),
21317        };
21318
21319        let commands = worker.execute_workflow_task(task).expect("workflow task");
21320
21321        assert_eq!(commands.len(), 1);
21322        assert_eq!(commands[0]["type"], "schedule_activity");
21323        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21324        assert_eq!(
21325            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21326            json!(["Rust"])
21327        );
21328    }
21329
21330    #[test]
21331    fn workflow_task_appends_paginated_history_events() {
21332        let mut task = WorkflowTask {
21333            task_id: "wft-rust-pages-1".to_string(),
21334            workflow_command_id: None,
21335            workflow_id: Some("wf-rust-pages".to_string()),
21336            run_id: Some("run-rust-pages".to_string()),
21337            workflow_type: "rust.hello_workflow".to_string(),
21338            cancel_requested: false,
21339            payload_codec: DEFAULT_CODEC.to_string(),
21340            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21341            history_events: vec![HistoryEvent {
21342                event_type: "WorkflowStarted".to_string(),
21343                payload: json!({}),
21344                raw: HashMap::new(),
21345            }],
21346            total_history_events: Some(3),
21347            history_size_bytes: None,
21348            continue_as_new_recommended: None,
21349            history_budget_pressure: None,
21350            next_history_page_token: Some("MQ==".to_string()),
21351            workflow_task_attempt: 1,
21352            workflow_signal_id: None,
21353            signal_name: None,
21354            signal_arguments: None,
21355            workflow_update_id: None,
21356            update_name: None,
21357            lease_owner: Some("rust-worker".to_string()),
21358        };
21359
21360        task.append_history_page(WorkflowTaskHistoryPage {
21361            history_events: vec![
21362                HistoryEvent {
21363                    event_type: "SignalReceived".to_string(),
21364                    payload: json!({
21365                        "signal_id": "sig-rust-1",
21366                        "signal_name": "start",
21367                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21368                            .expect("signal arguments")
21369                    }),
21370                    raw: HashMap::new(),
21371                },
21372                HistoryEvent {
21373                    event_type: "MarkerRecorded".to_string(),
21374                    payload: json!({"sequence": 3}),
21375                    raw: HashMap::new(),
21376                },
21377            ],
21378            total_history_events: Some(3),
21379            next_history_page_token: None,
21380        });
21381
21382        assert_eq!(task.history_events.len(), 3);
21383        assert_eq!(task.total_history_events, Some(3));
21384        assert_eq!(task.next_history_page_token, None);
21385
21386        let signal = task
21387            .history_events
21388            .iter()
21389            .find(|event| event.event_type == "SignalReceived")
21390            .expect("signal event");
21391        assert_eq!(
21392            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21393            vec![AvroValue::String("Rust".to_string())]
21394        );
21395    }
21396
21397    #[tokio::test]
21398    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21399        let client = Client::new("http://127.0.0.1:8080").expect("client");
21400        let mut worker = Worker::new(client, "rust-workers");
21401        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21402        worker.register_query("counter", "current", |ctx, _args| async move {
21403            let mut count = 0_i64;
21404            for signal in ctx.signal_events() {
21405                let value = signal
21406                    .arguments
21407                    .first()
21408                    .and_then(Value::as_i64)
21409                    .unwrap_or_default();
21410                match signal.name.as_str() {
21411                    "increment" => count += value,
21412                    "set" => count = value,
21413                    _ => {}
21414                }
21415            }
21416            Ok(json!(count))
21417        });
21418
21419        let task = QueryTask {
21420            query_task_id: "query-rust-counter".to_string(),
21421            query_task_attempt: 1,
21422            lease_owner: Some("rust-worker".to_string()),
21423            workflow_id: Some("counter-1".to_string()),
21424            run_id: Some("run-counter-1".to_string()),
21425            workflow_type: "counter".to_string(),
21426            query_name: "current".to_string(),
21427            payload_codec: DEFAULT_CODEC.to_string(),
21428            workflow_arguments: Some(
21429                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21430            ),
21431            query_arguments: Some(
21432                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21433            ),
21434            history_events: vec![
21435                HistoryEvent {
21436                    event_type: "SignalReceived".to_string(),
21437                    payload: json!({
21438                        "signal_id": "php-signal-1",
21439                        "signal_name": "increment",
21440                        "workflow_sequence": 1,
21441                        "payload_codec": DEFAULT_CODEC,
21442                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21443                    }),
21444                    raw: HashMap::new(),
21445                },
21446                HistoryEvent {
21447                    event_type: "SignalReceived".to_string(),
21448                    payload: json!({
21449                        "signal_id": "python-signal-2",
21450                        "signal_name": "increment",
21451                        "workflow_sequence": 2,
21452                        "payload_codec": DEFAULT_CODEC,
21453                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21454                    }),
21455                    raw: HashMap::new(),
21456                },
21457                HistoryEvent {
21458                    event_type: "SignalReceived".to_string(),
21459                    payload: json!({
21460                        "signal_id": "rust-signal-3",
21461                        "signal_name": "set",
21462                        "workflow_sequence": 3,
21463                        "payload_codec": DEFAULT_CODEC,
21464                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21465                    }),
21466                    raw: HashMap::new(),
21467                },
21468            ],
21469            history_export: None,
21470            run_status: Some("completed".to_string()),
21471        };
21472
21473        let result = worker.execute_query_task(task).await.expect("query result");
21474        assert_eq!(result.into_json().expect("query projection"), json!(0));
21475    }
21476
21477    #[tokio::test]
21478    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21479        let worker = replay_counter_worker();
21480        let running_history = json!([
21481            {
21482                "type": "ActivityCompleted",
21483                "payload": {
21484                    "sequence": 1,
21485                    "activity_type": "load-counter",
21486                    "payload_codec": DEFAULT_CODEC,
21487                    "result": fixture_envelope(json!("loaded"))
21488                }
21489            },
21490            {
21491                "type": "SignalWaitOpened",
21492                "payload": {
21493                    "sequence": 3,
21494                    "signal_name": "increment"
21495                }
21496            },
21497            {
21498                "type": "SignalReceived",
21499                "payload": {
21500                    "signal_id": "signal-3",
21501                    "signal_name": "increment",
21502                    "workflow_sequence": 2,
21503                    "payload_codec": DEFAULT_CODEC,
21504                    "arguments": fixture_envelope(json!([3]))
21505                }
21506            },
21507            {
21508                "type": "SignalApplied",
21509                "payload": {
21510                    "sequence": 3,
21511                    "signal_id": "signal-3",
21512                    "signal_name": "increment",
21513                    "payload_codec": DEFAULT_CODEC,
21514                    "value": fixture_envelope(json!([3]))
21515                }
21516            }
21517        ]);
21518
21519        let running = worker
21520            .execute_query_task(replay_counter_query(
21521                "current",
21522                running_history.clone(),
21523                "running",
21524            ))
21525            .await
21526            .expect("running replay query");
21527        assert_eq!(
21528            running.clone().into_json().expect("query projection"),
21529            json!({"loaded": "loaded", "count": 3, "finished": false})
21530        );
21531
21532        let detached = worker
21533            .execute_query_task(replay_counter_query(
21534                "detached-mutation",
21535                running_history.clone(),
21536                "running",
21537            ))
21538            .await
21539            .expect("query mutates only its detached state clone");
21540        assert_eq!(detached.into_json().expect("query projection"), json!(999));
21541        let failed = worker
21542            .execute_query_task(replay_counter_query(
21543                "failed-mutation",
21544                running_history.clone(),
21545                "running",
21546            ))
21547            .await
21548            .expect_err("failed query");
21549        assert_eq!(failed.reason, "query_rejected");
21550        let unchanged = worker
21551            .execute_query_task(replay_counter_query("current", running_history, "running"))
21552            .await
21553            .expect("later query reconstructs unchanged state");
21554        assert_eq!(unchanged, running);
21555
21556        let restarted_worker = replay_counter_worker();
21557        let empty_arguments = fixture_envelope(json!([]));
21558        let loaded_result = fixture_envelope(json!("loaded"));
21559        let signal_three = fixture_blob(json!([3]));
21560        let signal_five = fixture_blob(json!([5]));
21561        let restarted_task: QueryTask = serde_json::from_value(json!({
21562            "query_task_id": "query-after-restart",
21563            "workflow_id": "counter-1",
21564            "run_id": "run-counter-1",
21565            "workflow_type": "replay-counter",
21566            "query_name": "current",
21567            "payload_codec": DEFAULT_CODEC,
21568            "workflow_arguments": empty_arguments.clone(),
21569            "query_arguments": empty_arguments,
21570            "history_events": [],
21571            "history_export": {
21572                "payloads": {"codec": DEFAULT_CODEC},
21573                "history_events": [
21574                    {
21575                        "type": "ActivityCompleted",
21576                        "payload": {
21577                            "sequence": 1,
21578                            "activity_type": "load-counter",
21579                            "payload_codec": DEFAULT_CODEC,
21580                            "result": null
21581                        }
21582                    },
21583                    {
21584                        "type": "SignalWaitOpened",
21585                        "payload": {
21586                            "sequence": 3,
21587                            "signal_name": "increment"
21588                        }
21589                    },
21590                    {
21591                        "type": "SignalReceived",
21592                        "payload": {
21593                            "signal_id": "signal-3",
21594                            "signal_name": "increment",
21595                            "workflow_sequence": 2
21596                        }
21597                    },
21598                    {
21599                        "type": "SignalApplied",
21600                        "payload": {
21601                            "sequence": 3,
21602                            "signal_id": "signal-3",
21603                            "signal_name": "increment"
21604                        }
21605                    },
21606                    {
21607                        "type": "SignalWaitOpened",
21608                        "payload": {
21609                            "sequence": 5,
21610                            "signal_name": "increment"
21611                        }
21612                    },
21613                    {
21614                        "type": "SignalReceived",
21615                        "payload": {
21616                            "signal_id": "signal-5",
21617                            "signal_name": "increment",
21618                            "workflow_sequence": 4
21619                        }
21620                    },
21621                    {
21622                        "type": "SignalApplied",
21623                        "payload": {
21624                            "sequence": 5,
21625                            "signal_id": "signal-5",
21626                            "signal_name": "increment"
21627                        }
21628                    }
21629                ],
21630                "activities": [{
21631                    "sequence": 1,
21632                    "activity_type": "load-counter",
21633                    "payload_codec": DEFAULT_CODEC,
21634                    "result": loaded_result
21635                }],
21636                "signals": [
21637                    {
21638                        "id": "signal-3",
21639                        "name": "increment",
21640                        "workflow_sequence": 2,
21641                        "payload_codec": DEFAULT_CODEC,
21642                        "arguments": signal_three
21643                    },
21644                    {
21645                        "id": "signal-5",
21646                        "name": "increment",
21647                        "workflow_sequence": 4,
21648                        "payload_codec": DEFAULT_CODEC,
21649                        "arguments": signal_five
21650                    }
21651                ]
21652            },
21653            "run_status": "completed"
21654        }))
21655        .expect("cold replay query task");
21656        let completed = restarted_worker
21657            .execute_query_task(restarted_task)
21658            .await
21659            .expect("completed cold replay query");
21660        assert_eq!(
21661            completed.into_json().expect("query projection"),
21662            json!({"loaded": "loaded", "count": 8, "finished": true})
21663        );
21664    }
21665
21666    #[tokio::test]
21667    async fn replayed_query_replay_failures_are_machine_readable() {
21668        let worker = replay_counter_worker();
21669        let task = replay_counter_query(
21670            "current",
21671            json!([{
21672                "type": "ActivityCompleted",
21673                "payload": {
21674                    "sequence": 1,
21675                    "payload_codec": DEFAULT_CODEC,
21676                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21677                }
21678            }]),
21679            "running",
21680        );
21681        let failure = worker
21682            .execute_query_task(task)
21683            .await
21684            .expect_err("invalid replay history payload");
21685        assert_eq!(failure.reason, "query_payload_decode_failed");
21686        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21687        assert!(failure.message.contains("invalid_payload_framing"));
21688    }
21689
21690    #[tokio::test]
21691    async fn query_task_restores_compact_history_from_export() {
21692        let client = Client::new("http://127.0.0.1:8080").expect("client");
21693        let mut worker = Worker::new(client, "rust-workers");
21694        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21695        worker.register_query("counter", "current", |ctx, _args| async move {
21696            Ok(json!(ctx.signals("increment")[0][0]))
21697        });
21698        let empty_arguments = fixture_envelope(json!([]));
21699        let exported_signal = fixture_blob(json!([9]));
21700        let task: QueryTask = serde_json::from_value(json!({
21701            "query_task_id": "query-export",
21702            "workflow_type": "counter",
21703            "query_name": "current",
21704            "payload_codec": DEFAULT_CODEC,
21705            "workflow_arguments": empty_arguments.clone(),
21706            "query_arguments": empty_arguments,
21707            "history_events": [],
21708            "history_export": {
21709                "payloads": {"codec": DEFAULT_CODEC},
21710                "history_events": [{
21711                    "type": "SignalReceived",
21712                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21713                }],
21714                "signals": [{
21715                    "id": "signal-export",
21716                    "name": "increment",
21717                    "status": "applied",
21718                    "workflow_sequence": 1,
21719                    "payload_codec": DEFAULT_CODEC,
21720                    "arguments": exported_signal
21721                }]
21722            }
21723        }))
21724        .expect("query task");
21725
21726        let result = worker.execute_query_task(task).await.expect("query result");
21727        assert_eq!(result.into_json().expect("query projection"), json!(9));
21728    }
21729
21730    #[tokio::test]
21731    async fn query_task_failures_have_stable_reasons() {
21732        let client = Client::new("http://127.0.0.1:8080").expect("client");
21733        let mut worker = Worker::new(client, "rust-workers");
21734        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21735        worker.register_query(
21736            "counter",
21737            "current",
21738            |_ctx, _args| async move { Ok(json!(0)) },
21739        );
21740
21741        let base_task = QueryTask {
21742            query_task_id: "query-errors".to_string(),
21743            query_task_attempt: 1,
21744            lease_owner: None,
21745            workflow_id: Some("counter-errors".to_string()),
21746            run_id: Some("run-errors".to_string()),
21747            workflow_type: "counter".to_string(),
21748            query_name: "missing".to_string(),
21749            payload_codec: DEFAULT_CODEC.to_string(),
21750            workflow_arguments: Some(fixture_envelope(json!([]))),
21751            query_arguments: Some(fixture_envelope(json!([]))),
21752            history_events: Vec::new(),
21753            history_export: None,
21754            run_status: Some("running".to_string()),
21755        };
21756
21757        let unknown = worker
21758            .execute_query_task(base_task.clone())
21759            .await
21760            .expect_err("unknown query");
21761        assert_eq!(unknown.reason, "rejected_unknown_query");
21762
21763        let mut malformed = base_task;
21764        malformed.query_name = "current".to_string();
21765        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
21766        let malformed = worker
21767            .execute_query_task(malformed)
21768            .await
21769            .expect_err("malformed payload");
21770        assert_eq!(malformed.reason, "query_payload_decode_failed");
21771
21772        let client = Client::new("http://127.0.0.1:8080").expect("client");
21773        let mut unavailable_worker = Worker::new(client, "rust-workers");
21774        unavailable_worker
21775            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21776        let empty_arguments = fixture_envelope(json!([]));
21777        let unavailable_task: QueryTask = serde_json::from_value(json!({
21778            "query_task_id": "query-unavailable",
21779            "workflow_type": "counter",
21780            "query_name": "current",
21781            "payload_codec": DEFAULT_CODEC,
21782            "workflow_arguments": empty_arguments.clone(),
21783            "query_arguments": empty_arguments
21784        }))
21785        .expect("query task");
21786        let unavailable = unavailable_worker
21787            .execute_query_task(unavailable_task)
21788            .await
21789            .expect_err("query handler unavailable");
21790        assert_eq!(unavailable.reason, "query_handler_unavailable");
21791    }
21792
21793    #[tokio::test]
21794    async fn client_query_decodes_result_and_typed_failure() {
21795        let server = MockWorkerServer::start();
21796        let client = Client::builder(server.base_url())
21797            .timeout(Duration::from_secs(2))
21798            .build()
21799            .expect("client");
21800
21801        let result = client
21802            .query_workflow("counter-1", "current", json!([]))
21803            .await
21804            .expect("query result");
21805        assert_eq!(result, json!({"count": 8}));
21806
21807        let error = client
21808            .query_workflow("counter-1", "missing", json!([]))
21809            .await
21810            .expect_err("unknown query");
21811        let Error::QueryFailed(failure) = error else {
21812            panic!("expected typed query failure");
21813        };
21814        assert_eq!(failure.status, 404);
21815        assert_eq!(failure.reason, "rejected_unknown_query");
21816    }
21817
21818    #[tokio::test]
21819    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
21820        let server = MockWorkerServer::start();
21821        let client = Client::builder(server.base_url())
21822            .timeout(Duration::from_secs(2))
21823            .build()
21824            .expect("client");
21825        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
21826
21827        client
21828            .start_workflow(
21829                "typed.echo",
21830                "rust-workers",
21831                "typed-start",
21832                arguments.clone(),
21833            )
21834            .await
21835            .expect("typed workflow start");
21836        assert_eq!(
21837            decode_wire_avro_value(
21838                &server.request_body("/api/workflows")["input"],
21839                DEFAULT_CODEC,
21840            )
21841            .expect("typed start input"),
21842            arguments
21843        );
21844
21845        client
21846            .signal_workflow("typed-1", "changed", arguments.clone())
21847            .await
21848            .expect("typed signal");
21849        assert_eq!(
21850            decode_wire_avro_value(
21851                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
21852                DEFAULT_CODEC,
21853            )
21854            .expect("typed signal input"),
21855            arguments
21856        );
21857
21858        assert_eq!(
21859            client
21860                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
21861                .await
21862                .expect("typed query"),
21863            typed_fidelity_probe()
21864        );
21865        assert_eq!(
21866            decode_wire_avro_value(
21867                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
21868                DEFAULT_CODEC,
21869            )
21870            .expect("typed query input"),
21871            arguments
21872        );
21873
21874        assert_eq!(
21875            client
21876                .update_workflow_avro_value(
21877                    "typed-1",
21878                    "replace",
21879                    arguments.clone(),
21880                    Some("typed-request"),
21881                )
21882                .await
21883                .expect("typed update"),
21884            typed_fidelity_probe()
21885        );
21886        let update = server.request_body("/api/workflows/typed-1/update/replace");
21887        assert_eq!(update["request_id"], "typed-request");
21888        assert_eq!(
21889            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
21890            arguments
21891        );
21892
21893        let handle = WorkflowHandle {
21894            client: client.clone(),
21895            workflow_id: "typed-1".to_string(),
21896            run_id: Some("run-typed-1".to_string()),
21897            workflow_type: "typed.echo".to_string(),
21898        };
21899        assert_eq!(
21900            handle
21901                .result_avro_value(WorkflowResultOptions::default())
21902                .await
21903                .expect("typed workflow result"),
21904            typed_fidelity_probe()
21905        );
21906
21907        client
21908            .complete_activity_task(
21909                "activity-typed",
21910                "attempt-typed",
21911                "rust-worker",
21912                typed_fidelity_probe(),
21913                DEFAULT_CODEC,
21914            )
21915            .await
21916            .expect("typed activity completion");
21917        assert_eq!(
21918            decode_wire_avro_value(
21919                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
21920                    ["result"],
21921                DEFAULT_CODEC,
21922            )
21923            .expect("typed activity result"),
21924            typed_fidelity_probe()
21925        );
21926        client
21927            .fail_activity_task(
21928                "activity-typed",
21929                "attempt-typed",
21930                "rust-worker",
21931                "typed failure",
21932                true,
21933            )
21934            .await
21935            .expect("activity failure");
21936    }
21937
21938    #[tokio::test]
21939    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
21940        let server = MockWorkerServer::start();
21941        let client = Client::builder(server.base_url())
21942            .timeout(Duration::from_secs(2))
21943            .build()
21944            .expect("client");
21945
21946        let options = WorkflowCommandOptions::new()
21947            .reason("cleanup requested")
21948            .request_id("cancel-17");
21949        let cancelled = client
21950            .cancel_workflow("wf-lifecycle", options)
21951            .await
21952            .expect("instance cancellation");
21953        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
21954        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
21955        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
21956        assert_eq!(
21957            server.request_body("/api/workflows/wf-lifecycle/cancel"),
21958            json!({"reason":"cleanup requested","request_id":"cancel-17"})
21959        );
21960
21961        let terminated = client
21962            .terminate_workflow(
21963                "wf-lifecycle",
21964                WorkflowCommandOptions::new().reason("forced stop"),
21965            )
21966            .await
21967            .expect("instance termination");
21968        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
21969        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
21970
21971        client
21972            .cancel_workflow_run(
21973                "wf-lifecycle",
21974                "run-current",
21975                WorkflowCommandOptions::default(),
21976            )
21977            .await
21978            .expect("selected run cancellation");
21979        client
21980            .terminate_workflow_run(
21981                "wf-lifecycle",
21982                "run-current",
21983                WorkflowCommandOptions::default(),
21984            )
21985            .await
21986            .expect("selected run termination");
21987
21988        for (command, error) in [
21989            (
21990                WorkflowCommandKind::Cancel,
21991                client
21992                    .cancel_workflow_run(
21993                        "wf-lifecycle",
21994                        "run-stale",
21995                        WorkflowCommandOptions::default(),
21996                    )
21997                    .await
21998                    .expect_err("stale cancellation must be rejected"),
21999            ),
22000            (
22001                WorkflowCommandKind::Terminate,
22002                client
22003                    .terminate_workflow_run(
22004                        "wf-lifecycle",
22005                        "run-stale",
22006                        WorkflowCommandOptions::default(),
22007                    )
22008                    .await
22009                    .expect_err("stale termination must be rejected"),
22010            ),
22011        ] {
22012            let Error::WorkflowCommandRejected(rejection) = error else {
22013                panic!("expected typed command rejection");
22014            };
22015            assert_eq!(rejection.command, command);
22016            assert_eq!(rejection.status, 409);
22017            assert_eq!(rejection.reason, "historical_run_command_rejected");
22018            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22019            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22020        }
22021    }
22022
22023    #[tokio::test]
22024    async fn workflow_start_options_send_server_enforced_deadlines() {
22025        let server = MockWorkerServer::start();
22026        let client = Client::builder(server.base_url())
22027            .timeout(Duration::from_secs(2))
22028            .build()
22029            .expect("client");
22030
22031        let handle = client
22032            .start_workflow_with_options(
22033                "rust.timeout",
22034                "rust-timeouts",
22035                "wf-start-options",
22036                WorkflowStartOptions::new()
22037                    .execution_timeout_seconds(30)
22038                    .run_timeout_seconds(1),
22039                json!([]),
22040            )
22041            .await
22042            .expect("workflow start");
22043
22044        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22045        let body = server.request_body("/api/workflows");
22046        assert_eq!(body["execution_timeout_seconds"], 30);
22047        assert_eq!(body["run_timeout_seconds"], 1);
22048
22049        let invalid = client
22050            .start_workflow_with_options(
22051                "rust.timeout",
22052                "rust-timeouts",
22053                "wf-invalid-options",
22054                WorkflowStartOptions::new()
22055                    .execution_timeout_seconds(1)
22056                    .run_timeout_seconds(2),
22057                json!([]),
22058            )
22059            .await
22060            .expect_err("invalid deadline ordering");
22061        assert!(invalid
22062            .to_string()
22063            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22064    }
22065
22066    #[tokio::test]
22067    async fn workflow_result_returns_each_typed_terminal_outcome() {
22068        let server = MockWorkerServer::start();
22069        let client = Client::builder(server.base_url())
22070            .timeout(Duration::from_secs(2))
22071            .build()
22072            .expect("client");
22073        let options = WorkflowResultOptions {
22074            poll_interval: Duration::ZERO,
22075            timeout: Duration::from_secs(1),
22076        };
22077
22078        let failed = WorkflowHandle {
22079            client: client.clone(),
22080            workflow_id: "wf-failed".to_string(),
22081            run_id: Some("run-failed".to_string()),
22082            workflow_type: "failure".to_string(),
22083        }
22084        .result(options)
22085        .await
22086        .expect_err("failed outcome");
22087        let Error::WorkflowFailed(failure) = failed else {
22088            panic!("expected WorkflowFailed");
22089        };
22090        assert_eq!(failure.workflow_id, "wf-failed");
22091        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22092        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22093        assert_eq!(failure.failure_category.as_deref(), Some("application"));
22094        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22095        assert_eq!(
22096            failure.exception_class.as_deref(),
22097            Some("billing::PaymentError")
22098        );
22099        assert_eq!(failure.non_retryable, Some(true));
22100
22101        for (workflow_id, expected_kind, expected_reason) in [
22102            (
22103                "wf-cancelled",
22104                WorkflowTerminalKind::Cancelled,
22105                "cleanup requested",
22106            ),
22107            (
22108                "wf-terminated",
22109                WorkflowTerminalKind::Terminated,
22110                "forced stop",
22111            ),
22112            (
22113                "wf-timed-out",
22114                WorkflowTerminalKind::TimedOut,
22115                "run_timeout",
22116            ),
22117        ] {
22118            let error = WorkflowHandle {
22119                client: client.clone(),
22120                workflow_id: workflow_id.to_string(),
22121                run_id: None,
22122                workflow_type: "terminal".to_string(),
22123            }
22124            .result(options)
22125            .await
22126            .expect_err("typed terminal outcome");
22127            let outcome = match error {
22128                Error::WorkflowCancelled(outcome) => outcome,
22129                Error::WorkflowTerminated(outcome) => outcome,
22130                Error::WorkflowTimedOut(outcome) => outcome,
22131                other => panic!("unexpected terminal error: {other}"),
22132            };
22133            assert_eq!(outcome.kind, expected_kind);
22134            assert_eq!(outcome.workflow_id, workflow_id);
22135            assert_eq!(outcome.reason, expected_reason);
22136        }
22137
22138        let wait_timeout = WorkflowHandle {
22139            client,
22140            workflow_id: "wf-waiting".to_string(),
22141            run_id: Some("run-waiting".to_string()),
22142            workflow_type: "waiting".to_string(),
22143        }
22144        .result(WorkflowResultOptions {
22145            poll_interval: Duration::ZERO,
22146            timeout: Duration::ZERO,
22147        })
22148        .await
22149        .expect_err("client wait timeout");
22150        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22151            panic!("expected typed client timeout");
22152        };
22153        assert_eq!(timeout.reason, "result_wait_timeout");
22154        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22155        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22156    }
22157
22158    #[tokio::test]
22159    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22160        let server = MockWorkerServer::start();
22161        let client = Client::builder(server.base_url())
22162            .timeout(Duration::from_secs(2))
22163            .build()
22164            .expect("client");
22165
22166        let handle = WorkflowHandle {
22167            client,
22168            workflow_id: "wf-selected".to_string(),
22169            run_id: Some("run-selected".to_string()),
22170            workflow_type: "selected".to_string(),
22171        };
22172        let options = WorkflowResultOptions {
22173            poll_interval: Duration::ZERO,
22174            timeout: Duration::from_secs(1),
22175        };
22176
22177        let current = handle
22178            .result(options)
22179            .await
22180            .expect("instance result follows the current run");
22181        assert_eq!(current, json!("current run output"));
22182
22183        let error = handle
22184            .result_selected_run(options)
22185            .await
22186            .expect_err("the selected run is cancelled even though the current run completed");
22187
22188        let Error::WorkflowCancelled(outcome) = error else {
22189            panic!("expected selected run cancellation");
22190        };
22191        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22192        assert_eq!(outcome.reason, "selected run cancelled");
22193        assert_eq!(
22194            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22195            1
22196        );
22197        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22198    }
22199
22200    #[tokio::test]
22201    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22202        let server = MockWorkerServer::draining_polls();
22203        let client = Client::builder(server.base_url())
22204            .timeout(Duration::from_secs(2))
22205            .build()
22206            .expect("client");
22207
22208        let workflow = client
22209            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22210            .await
22211            .expect("workflow drain response");
22212        let activity = client
22213            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22214            .await
22215            .expect("activity drain response");
22216        let query = client
22217            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22218            .await
22219            .expect("query drain response");
22220
22221        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22222            assert_eq!(
22223                outcome,
22224                WorkerPollOutcome::Stop {
22225                    poll_status: Some("draining".to_string()),
22226                    reason: Some("worker_draining".to_string()),
22227                }
22228            );
22229        }
22230
22231        assert!(client
22232            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22233            .await
22234            .expect("compatibility poll")
22235            .is_none());
22236    }
22237
22238    #[tokio::test]
22239    async fn managed_worker_honors_drain_stop_for_every_task_family() {
22240        let server = MockWorkerServer::draining_polls();
22241        let client = Client::builder(server.base_url())
22242            .timeout(Duration::from_secs(2))
22243            .build()
22244            .expect("client");
22245
22246        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22247            .worker_id("draining-workflow-worker")
22248            .poll_timeout(Duration::ZERO);
22249        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22250        workflow_worker
22251            .run()
22252            .await
22253            .expect("workflow drain is a clean stop");
22254
22255        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22256            .worker_id("draining-activity-worker")
22257            .poll_timeout(Duration::ZERO);
22258        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22259        activity_worker
22260            .run()
22261            .await
22262            .expect("activity drain is a clean stop");
22263
22264        let mut query_worker = Worker::new(client, "rust-workers")
22265            .worker_id("draining-query-worker")
22266            .poll_timeout(Duration::ZERO);
22267        query_worker.register_query("counter", "current", |_ctx, _args| async {
22268            Ok(Value::Null)
22269        });
22270        query_worker
22271            .run()
22272            .await
22273            .expect("query drain is a clean stop");
22274    }
22275
22276    #[tokio::test]
22277    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22278        let server = MockWorkerServer::start();
22279        let client = Client::builder(server.base_url())
22280            .timeout(Duration::from_secs(2))
22281            .build()
22282            .expect("client");
22283
22284        let heartbeat = client
22285            .heartbeat_activity_task(
22286                "activity-cancel",
22287                "attempt-cancel",
22288                "rust-worker",
22289                typed_fidelity_probe(),
22290            )
22291            .await
22292            .expect("cancellation heartbeat");
22293        assert!(heartbeat.cancel_requested);
22294        assert!(heartbeat.should_stop());
22295        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22296        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22297        let heartbeat_body =
22298            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22299        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22300        assert_eq!(
22301            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22302                .expect("typed heartbeat details"),
22303            typed_fidelity_probe()
22304        );
22305
22306        let error = client
22307            .complete_activity_task(
22308                "activity-cancel",
22309                "attempt-cancel",
22310                "rust-worker",
22311                json!({"late":true}),
22312                DEFAULT_CODEC,
22313            )
22314            .await
22315            .expect_err("late completion must be refused");
22316        assert!(activity_task_rejection_is_final(&error));
22317        let Error::ActivityTaskRejected(rejection) = error else {
22318            panic!("expected typed activity rejection");
22319        };
22320        assert_eq!(rejection.status, 409);
22321        assert_eq!(rejection.reason, "run_cancelled");
22322        assert!(rejection.cancel_requested);
22323        assert_eq!(rejection.can_continue, Some(false));
22324    }
22325
22326    #[tokio::test]
22327    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22328        let server = MockWorkerServer::cancelled_activity();
22329        let client = Client::builder(server.base_url())
22330            .timeout(Duration::from_secs(2))
22331            .build()
22332            .expect("client");
22333        let cancellation_observed = Arc::new(AtomicBool::new(false));
22334        let observed = Arc::clone(&cancellation_observed);
22335        let mut worker = Worker::new(client.clone(), "rust-workers")
22336            .worker_id("rust-cancel-worker")
22337            .poll_timeout(Duration::from_millis(10));
22338        worker.register_activity("cancel-aware", move |ctx, _args| {
22339            let observed = Arc::clone(&observed);
22340            async move {
22341                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22342                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22343                Ok(json!({"late":"completion"}))
22344            }
22345        });
22346
22347        assert_eq!(
22348            worker.run_once().await.expect("cancelled attempt handled"),
22349            1
22350        );
22351        assert!(cancellation_observed.load(Ordering::SeqCst));
22352        assert_eq!(
22353            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22354            1
22355        );
22356
22357        let mut restarted = Worker::new(client, "rust-workers")
22358            .worker_id("rust-cancel-worker-restarted")
22359            .poll_timeout(Duration::from_millis(10));
22360        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22361        assert_eq!(
22362            restarted
22363                .run_once()
22364                .await
22365                .expect("replacement worker continues polling"),
22366            0
22367        );
22368    }
22369
22370    #[tokio::test]
22371    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22372        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22373        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22374        let client = Client::builder(server.base_url())
22375            .timeout(Duration::from_secs(2))
22376            .build()
22377            .expect("client");
22378
22379        let direct_error = client
22380            .complete_workflow_task(
22381                "workflow-timeout-task",
22382                "timeout-worker",
22383                3,
22384                vec![json!({
22385                    "type": "complete_workflow",
22386                    "result": fixture_envelope(Value::Null)
22387                })],
22388            )
22389            .await
22390            .expect_err("the low-level client preserves the completion rejection");
22391        let Error::Http { status, body } = direct_error else {
22392            panic!("expected the original HTTP completion rejection");
22393        };
22394        assert_eq!(status, reqwest::StatusCode::CONFLICT);
22395        assert_eq!(
22396            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22397            "run_timed_out"
22398        );
22399
22400        let mut worker = Worker::new(client, "rust-workers")
22401            .worker_id("timeout-worker")
22402            .poll_timeout(Duration::from_millis(10));
22403        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22404            Ok(json!({"late": "result"}))
22405        });
22406
22407        assert_eq!(
22408            worker
22409                .run_once()
22410                .await
22411                .expect("authoritative selected-run timeout settles the tick"),
22412            1
22413        );
22414        assert_eq!(
22415            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22416            2,
22417            "both the direct client proof and managed worker must see the rejection"
22418        );
22419    }
22420
22421    #[tokio::test]
22422    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22423        for (name, status, response) in [
22424            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22425            (
22426                "command was recorded",
22427                "409 Conflict",
22428                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22429            ),
22430            (
22431                "lease conflict",
22432                "409 Conflict",
22433                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22434            ),
22435            (
22436                "nonterminal run",
22437                "409 Conflict",
22438                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22439            ),
22440            (
22441                "different selected run",
22442                "409 Conflict",
22443                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22444            ),
22445            (
22446                "different task attempt",
22447                "409 Conflict",
22448                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22449            ),
22450            (
22451                "authentication failure",
22452                "401 Unauthorized",
22453                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22454            ),
22455            (
22456                "authorization failure",
22457                "403 Forbidden",
22458                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22459            ),
22460            (
22461                "protocol failure",
22462                "400 Bad Request",
22463                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22464            ),
22465            (
22466                "malformed command",
22467                "422 Unprocessable Entity",
22468                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22469            ),
22470            (
22471                "transient server failure",
22472                "503 Service Unavailable",
22473                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22474            ),
22475        ] {
22476            let server = MockWorkerServer::workflow_completion(status, response);
22477            let client = Client::builder(server.base_url())
22478                .timeout(Duration::from_secs(2))
22479                .build()
22480                .expect("client");
22481            let mut worker = Worker::new(client, "rust-workers")
22482                .worker_id("timeout-worker")
22483                .poll_timeout(Duration::from_millis(10));
22484            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22485                Ok(json!({"late": "result"}))
22486            });
22487
22488            let error = worker
22489                .run_once()
22490                .await
22491                .expect_err(&format!("{name} must remain an error"));
22492            assert!(
22493                matches!(error, Error::Http { .. } | Error::Protocol(_)),
22494                "{name} returned an unexpected error variant: {error}"
22495            );
22496        }
22497    }
22498
22499    #[tokio::test]
22500    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22501        let server = MockWorkerServer::start();
22502        let client = Client::builder(server.base_url())
22503            .worker_token(Some("worker-secret".to_string()))
22504            .namespace("orders")
22505            .timeout(Duration::from_secs(2))
22506            .build()
22507            .expect("client");
22508        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22509
22510        let result = client
22511            .deregister_worker_registration("worker/α space")
22512            .await
22513            .expect("deregister worker registration");
22514
22515        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22516        assert_eq!(
22517            server.worker_protocol_for(path).as_deref(),
22518            Some(WORKER_PROTOCOL_VERSION)
22519        );
22520        assert_eq!(server.control_protocol_for(path), None);
22521        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22522        assert_eq!(
22523            server.authorization_for(path).as_deref(),
22524            Some("Bearer worker-secret")
22525        );
22526        assert_eq!(
22527            result,
22528            WorkerDeregistrationEnvelope {
22529                worker_id: "deregistered-worker".to_string(),
22530                outcome: "deregistered".to_string(),
22531                recovered_workflow_task_count: 2,
22532            }
22533        );
22534    }
22535
22536    #[tokio::test]
22537    async fn low_level_registration_rejects_update_validators_before_transport() {
22538        let server = MockWorkerServer::start();
22539        let client = Client::builder(server.base_url())
22540            .timeout(Duration::from_secs(2))
22541            .build()
22542            .expect("client");
22543
22544        for update_validators in [json!(["approve"]), json!("approve")] {
22545            let error = client
22546                .register_worker_with_command_contracts(
22547                    "validator-claiming-worker",
22548                    "rust-workers",
22549                    vec!["orders".to_string()],
22550                    vec![],
22551                    1,
22552                    1,
22553                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22554                    json!({
22555                        "orders": {
22556                            "queries": ["current"],
22557                            "updates": ["approve"],
22558                            "update_validators": update_validators,
22559                        },
22560                    }),
22561                )
22562                .await
22563                .expect_err("unsupported validator claims must fail before registration");
22564
22565            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22566                panic!("expected typed unsupported-validator failure");
22567            };
22568            assert_eq!(workflow_type, "orders");
22569        }
22570        assert_eq!(server.request_count("/api/worker/register"), 0);
22571    }
22572
22573    #[tokio::test]
22574    async fn low_level_registration_preserves_query_and_update_contracts() {
22575        let server = MockWorkerServer::start();
22576        let client = Client::builder(server.base_url())
22577            .timeout(Duration::from_secs(2))
22578            .build()
22579            .expect("client");
22580        let contracts = json!({
22581            "orders": {
22582                "queries": ["current"],
22583                "updates": ["approve"],
22584                "update_validators": [],
22585            },
22586            "payments": {
22587                "queries": ["status"],
22588                "updates": ["capture"],
22589            },
22590        });
22591
22592        client
22593            .register_worker_with_command_contracts(
22594                "command-worker",
22595                "rust-workers",
22596                vec!["orders".to_string(), "payments".to_string()],
22597                vec![],
22598                1,
22599                1,
22600                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22601                contracts.clone(),
22602            )
22603            .await
22604            .expect("query and update contracts must remain supported");
22605
22606        assert_eq!(
22607            server.request_body("/api/worker/register")["workflow_command_contracts"],
22608            contracts
22609        );
22610    }
22611
22612    #[tokio::test]
22613    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22614        let server = MockWorkerServer::start();
22615        let control_only = Client::builder(server.base_url())
22616            .control_token(Some("control-secret".to_string()))
22617            .build()
22618            .expect("control client");
22619
22620        let error = control_only
22621            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22622            .await
22623            .expect_err("control token must not authorize a worker request");
22624        assert!(matches!(
22625            error,
22626            Error::MissingRoleCredentials { role: "worker", .. }
22627        ));
22628        assert_eq!(server.request_count("/api/worker/register"), 0);
22629
22630        let worker_only = Client::builder(server.base_url())
22631            .worker_token(Some("worker-secret".to_string()))
22632            .build()
22633            .expect("worker client");
22634        let error = worker_only
22635            .health()
22636            .await
22637            .expect_err("worker token must not authorize a control request");
22638        assert!(matches!(
22639            error,
22640            Error::MissingRoleCredentials {
22641                role: "control",
22642                ..
22643            }
22644        ));
22645        assert_eq!(server.request_count("/api/health"), 0);
22646    }
22647
22648    #[tokio::test]
22649    async fn shared_token_supports_worker_and_control_planes() {
22650        let server = MockWorkerServer::start();
22651        let client = Client::builder(server.base_url())
22652            .token(Some("shared-secret".to_string()))
22653            .build()
22654            .expect("client");
22655
22656        client.health().await.expect("control request");
22657        client
22658            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22659            .await
22660            .expect("worker request");
22661
22662        assert_eq!(
22663            server.authorization_for("/api/health").as_deref(),
22664            Some("Bearer shared-secret")
22665        );
22666        assert_eq!(
22667            server.control_protocol_for("/api/health").as_deref(),
22668            Some(CONTROL_PLANE_VERSION)
22669        );
22670        assert_eq!(
22671            server.authorization_for("/api/worker/register").as_deref(),
22672            Some("Bearer shared-secret")
22673        );
22674        assert_eq!(
22675            server
22676                .worker_protocol_for("/api/worker/register")
22677                .as_deref(),
22678            Some(WORKER_PROTOCOL_VERSION)
22679        );
22680    }
22681
22682    #[tokio::test]
22683    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22684        let server = MockWorkerServer::start();
22685        let client = Client::builder(server.base_url())
22686            .timeout(Duration::from_secs(2))
22687            .build()
22688            .expect("client");
22689
22690        client
22691            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22692            .await
22693            .expect("register");
22694        client
22695            .heartbeat_worker("capture-worker", 1, 1)
22696            .await
22697            .expect("heartbeat");
22698        client
22699            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22700            .await
22701            .expect("workflow poll");
22702        client
22703            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22704            .await
22705            .expect("activity poll");
22706
22707        for path in [
22708            "/api/worker/register",
22709            "/api/worker/heartbeat",
22710            "/api/worker/workflow-tasks/poll",
22711            "/api/worker/activity-tasks/poll",
22712        ] {
22713            assert_eq!(
22714                server.worker_protocol_for(path).as_deref(),
22715                Some(WORKER_PROTOCOL_VERSION),
22716                "unexpected protocol for {path}"
22717            );
22718        }
22719
22720        assert_eq!(
22721            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22722            1
22723        );
22724        assert_eq!(
22725            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22726            1
22727        );
22728        assert!(
22729            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22730                .as_str()
22731                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22732        );
22733        assert!(
22734            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22735                .as_str()
22736                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22737        );
22738    }
22739
22740    #[tokio::test]
22741    async fn query_task_endpoints_send_the_query_feature_protocol() {
22742        let server = MockWorkerServer::start();
22743        let client = Client::builder(server.base_url())
22744            .timeout(Duration::from_secs(2))
22745            .build()
22746            .expect("client");
22747
22748        client
22749            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22750            .await
22751            .expect("query poll");
22752        client
22753            .complete_query_task(
22754                "query-capture",
22755                "capture-worker",
22756                1,
22757                json!(8),
22758                DEFAULT_CODEC,
22759            )
22760            .await
22761            .expect("query complete");
22762        client
22763            .fail_query_task(
22764                "query-capture",
22765                "capture-worker",
22766                1,
22767                "failed",
22768                "query_rejected",
22769                "QueryFailed",
22770            )
22771            .await
22772            .expect("query fail");
22773
22774        for path in [
22775            "/api/worker/query-tasks/poll",
22776            "/api/worker/query-tasks/query-capture/complete",
22777            "/api/worker/query-tasks/query-capture/fail",
22778        ] {
22779            assert_eq!(
22780                server.worker_protocol_for(path).as_deref(),
22781                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
22782                "unexpected protocol for {path}"
22783            );
22784        }
22785
22786        assert_eq!(
22787            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
22788            1
22789        );
22790        assert!(
22791            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
22792                .as_str()
22793                .is_some_and(|id| id.starts_with("rust-query-poll-"))
22794        );
22795    }
22796
22797    #[tokio::test]
22798    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
22799        let server = MockWorkerServer::transient_worker_failures();
22800        let client = Client::builder(server.base_url())
22801            .timeout(Duration::from_secs(2))
22802            .build()
22803            .expect("client");
22804
22805        client
22806            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22807            .await
22808            .expect("workflow poll retry");
22809        client
22810            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22811            .await
22812            .expect("activity poll retry");
22813        client
22814            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22815            .await
22816            .expect("query poll retry");
22817
22818        for path in [
22819            "/api/worker/workflow-tasks/poll",
22820            "/api/worker/activity-tasks/poll",
22821            "/api/worker/query-tasks/poll",
22822        ] {
22823            let bodies = server.request_bodies(path);
22824            assert_eq!(bodies.len(), 2, "{path} must be retried once");
22825            assert_eq!(
22826                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
22827                "{path} must preserve the request binding across retry"
22828            );
22829        }
22830    }
22831
22832    #[tokio::test]
22833    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
22834        let server = MockWorkerServer::consecutive_poll_failures(2);
22835        let client = Client::builder(server.base_url())
22836            .timeout(Duration::from_secs(2))
22837            .build()
22838            .expect("client");
22839        let mut worker = Worker::new(client, "capture")
22840            .worker_id("capture-worker")
22841            .poll_timeout(Duration::from_millis(10))
22842            .retry_policy(WorkerRetryPolicy {
22843                max_retries: 2,
22844                initial_backoff: Duration::from_millis(1),
22845                max_backoff: Duration::from_millis(1),
22846            });
22847        worker.register_workflow(
22848            "capture.workflow",
22849            |_ctx, _input| async move { Ok(Value::Null) },
22850        );
22851        worker.register_activity(
22852            "capture.activity",
22853            |_ctx, _input| async move { Ok(Value::Null) },
22854        );
22855        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
22856            Ok(Value::Null)
22857        });
22858
22859        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
22860
22861        for path in [
22862            "/api/worker/workflow-tasks/poll",
22863            "/api/worker/activity-tasks/poll",
22864            "/api/worker/query-tasks/poll",
22865        ] {
22866            let bodies = server.request_bodies(path);
22867            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
22868            assert!(
22869                bodies
22870                    .iter()
22871                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
22872                "{path} must preserve one request binding across every retry"
22873            );
22874        }
22875    }
22876
22877    #[tokio::test]
22878    async fn query_protocol_rejection_from_older_server_is_typed() {
22879        let server = MockWorkerServer::reject_query_protocol();
22880        let client = Client::builder(server.base_url())
22881            .timeout(Duration::from_secs(2))
22882            .build()
22883            .expect("client");
22884
22885        let error = client
22886            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22887            .await
22888            .expect_err("server below query protocol floor must reject");
22889        let Error::Protocol(failure) = error else {
22890            panic!("expected typed protocol failure");
22891        };
22892
22893        assert_eq!(failure.status, 400);
22894        assert_eq!(failure.reason, "unsupported_protocol_version");
22895        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
22896        assert_eq!(
22897            failure.requested_version.as_deref(),
22898            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
22899        );
22900        assert_eq!(
22901            server
22902                .worker_protocol_for("/api/worker/query-tasks/poll")
22903                .as_deref(),
22904            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
22905        );
22906    }
22907
22908    #[tokio::test]
22909    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
22910        let server = MockWorkerServer::reject_query_protocol();
22911        let client = Client::builder(server.base_url())
22912            .timeout(Duration::from_secs(2))
22913            .build()
22914            .expect("client");
22915        let mut worker = Worker::new(client, "rust-workers")
22916            .worker_id("baseline-worker")
22917            .poll_timeout(Duration::from_millis(10));
22918
22919        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
22920            Ok(Value::Null)
22921        });
22922
22923        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
22924        assert_eq!(
22925            server
22926                .worker_protocol_for("/api/worker/workflow-tasks/poll")
22927                .as_deref(),
22928            Some(WORKER_PROTOCOL_VERSION)
22929        );
22930        assert_eq!(
22931            server.worker_protocol_for("/api/worker/query-tasks/poll"),
22932            None,
22933            "a worker without query handlers must not use the query-task endpoint"
22934        );
22935    }
22936
22937    #[tokio::test]
22938    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
22939        let server = MockWorkerServer::reject_query_completion();
22940        let client = Client::builder(server.base_url())
22941            .timeout(Duration::from_secs(2))
22942            .build()
22943            .expect("client");
22944
22945        let error = client
22946            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
22947            .await
22948            .expect_err("expired completion must be rejected");
22949        let Error::QueryFailed(failure) = error else {
22950            panic!("expected typed query failure");
22951        };
22952        assert_eq!(failure.status, 409);
22953        assert_eq!(failure.reason, "query_task_timed_out");
22954
22955        let mut worker = Worker::new(client, "rust-workers")
22956            .worker_id("late-worker")
22957            .poll_timeout(Duration::from_millis(10));
22958        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22959        worker.register_query(
22960            "counter",
22961            "current",
22962            |_ctx, _args| async move { Ok(json!(8)) },
22963        );
22964
22965        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
22966        assert_eq!(
22967            worker
22968                .run_once()
22969                .await
22970                .expect("worker continues after late completion"),
22971            0
22972        );
22973        assert_eq!(
22974            server.request_count("/api/worker/query-tasks/query-late/complete"),
22975            2
22976        );
22977        assert_eq!(
22978            server.request_count("/api/worker/query-tasks/query-late/fail"),
22979            0,
22980            "a server completion rejection must not be reported as an encoding failure"
22981        );
22982    }
22983
22984    #[tokio::test]
22985    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
22986        let server = MockWorkerServer::start();
22987        let client = Client::builder(server.base_url())
22988            .timeout(Duration::from_secs(2))
22989            .build()
22990            .expect("client");
22991        let mut worker = Worker::new(client, "rust-workers")
22992            .worker_id("joined-worker")
22993            .poll_timeout(Duration::from_millis(10));
22994        worker.register_workflow(
22995            "joined.workflow",
22996            |_ctx, _input| async move { Ok(Value::Null) },
22997        );
22998        worker.register_activity(
22999            "joined.activity",
23000            |_ctx, _input| async move { Ok(Value::Null) },
23001        );
23002        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
23003            Ok(Value::Null)
23004        });
23005
23006        worker
23007            .run_until(tokio::time::sleep(Duration::from_millis(20)))
23008            .await
23009            .expect("normal shutdown");
23010
23011        let deregistration_path = "/api/worker/registrations/mock-worker";
23012        assert_eq!(server.request_count(deregistration_path), 1);
23013        for poll_path in [
23014            "/api/worker/workflow-tasks/poll",
23015            "/api/worker/activity-tasks/poll",
23016            "/api/worker/query-tasks/poll",
23017        ] {
23018            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
23019        }
23020        assert_eq!(
23021            server.captured_paths().last().map(String::as_str),
23022            Some(deregistration_path),
23023            "deregistration must start only after every poller has joined"
23024        );
23025    }
23026
23027    #[tokio::test]
23028    async fn registration_failure_does_not_deregister() {
23029        let server = MockWorkerServer::rejected_registration();
23030        let client = Client::builder(server.base_url())
23031            .timeout(Duration::from_secs(2))
23032            .build()
23033            .expect("client");
23034        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
23035
23036        let error = worker
23037            .run_until(async {})
23038            .await
23039            .expect_err("registration must fail");
23040        assert!(matches!(
23041            error,
23042            Error::Http {
23043                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23044                ..
23045            }
23046        ));
23047        assert!(server
23048            .captured_paths()
23049            .iter()
23050            .all(|path| !path.starts_with("/api/worker/registrations/")));
23051    }
23052
23053    #[tokio::test]
23054    async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
23055        let server = MockWorkerServer::rejected_registration_protocol();
23056        let client = Client::builder(server.base_url())
23057            .timeout(Duration::from_secs(2))
23058            .build()
23059            .expect("client");
23060        let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
23061
23062        let error = worker
23063            .run_until(async {})
23064            .await
23065            .expect_err("a protocol 1.16 server must reject this worker");
23066        let Error::Protocol(failure) = error else {
23067            panic!("expected typed protocol rejection");
23068        };
23069        assert_eq!(failure.reason, "unsupported_protocol_version");
23070        assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
23071        assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
23072        assert_eq!(
23073            server
23074                .worker_protocol_for("/api/worker/register")
23075                .as_deref(),
23076            Some(WORKER_PROTOCOL_VERSION)
23077        );
23078    }
23079
23080    #[tokio::test]
23081    async fn declined_registration_does_not_deregister() {
23082        let server = MockWorkerServer::declined_registration();
23083        let client = Client::builder(server.base_url())
23084            .timeout(Duration::from_secs(2))
23085            .build()
23086            .expect("client");
23087        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
23088
23089        let error = worker
23090            .run_until(async {})
23091            .await
23092            .expect_err("declined registration must fail");
23093        assert!(matches!(error, Error::WorkerLoop(_)));
23094        assert!(error.to_string().contains("was not accepted"));
23095        assert!(server
23096            .captured_paths()
23097            .iter()
23098            .all(|path| !path.starts_with("/api/worker/registrations/")));
23099    }
23100
23101    #[tokio::test]
23102    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23103        let server = MockWorkerServer::rejected_deregistration();
23104        let client = Client::builder(server.base_url())
23105            .timeout(Duration::from_secs(2))
23106            .build()
23107            .expect("client");
23108        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23109
23110        let error = worker
23111            .run_until(async {})
23112            .await
23113            .expect_err("deregistration must fail");
23114        assert!(matches!(
23115            error,
23116            Error::Http {
23117                status: reqwest::StatusCode::FORBIDDEN,
23118                ..
23119            }
23120        ));
23121        assert_eq!(
23122            server.request_count("/api/worker/registrations/mock-worker"),
23123            1
23124        );
23125    }
23126
23127    #[tokio::test]
23128    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23129        let server = MockWorkerServer::rejected_deregistration_protocol();
23130        let client = Client::builder(server.base_url())
23131            .timeout(Duration::from_secs(2))
23132            .build()
23133            .expect("client");
23134        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
23135
23136        let error = worker
23137            .run_until(async {})
23138            .await
23139            .expect_err("protocol rejection must fail shutdown");
23140        let Error::Protocol(failure) = error else {
23141            panic!("expected typed protocol failure");
23142        };
23143        assert_eq!(failure.reason, "unsupported_protocol_version");
23144        assert_eq!(
23145            failure.requested_version.as_deref(),
23146            Some(WORKER_PROTOCOL_VERSION)
23147        );
23148        assert_eq!(
23149            server.request_count("/api/worker/registrations/mock-worker"),
23150            1
23151        );
23152    }
23153
23154    #[tokio::test]
23155    async fn primary_poller_error_retains_deregistration_failure_context() {
23156        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
23157        let client = Client::builder(server.base_url())
23158            .timeout(Duration::from_secs(2))
23159            .build()
23160            .expect("client");
23161        let mut worker = Worker::new(client, "rust-workers")
23162            .worker_id("combined-failure")
23163            .poll_timeout(Duration::from_millis(10));
23164        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
23165            Ok(Value::Null)
23166        });
23167
23168        let error = worker
23169            .run()
23170            .await
23171            .expect_err("worker and cleanup must fail");
23172        let summary = error.to_string();
23173        assert!(summary.contains("authentication_failed"));
23174        assert!(summary.contains("worker cannot deregister"));
23175        let Error::WorkerShutdown {
23176            primary,
23177            deregistration,
23178        } = error
23179        else {
23180            panic!("expected combined worker shutdown error");
23181        };
23182        assert!(matches!(
23183            *primary,
23184            Error::Http {
23185                status: reqwest::StatusCode::UNAUTHORIZED,
23186                ..
23187            }
23188        ));
23189        assert!(matches!(
23190            *deregistration,
23191            Error::Http {
23192                status: reqwest::StatusCode::FORBIDDEN,
23193                ..
23194            }
23195        ));
23196        assert_eq!(
23197            server.request_count("/api/worker/registrations/mock-worker"),
23198            1
23199        );
23200    }
23201
23202    #[tokio::test]
23203    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
23204        let server = MockWorkerServer::start();
23205        let client = Client::builder(server.base_url())
23206            .timeout(Duration::from_secs(2))
23207            .build()
23208            .expect("client");
23209        let mut worker = Worker::new(client, "rust-workers")
23210            .worker_id("activity-only-worker")
23211            .poll_timeout(Duration::from_millis(10));
23212
23213        worker.register_activity(
23214            "activity.only",
23215            |_ctx, _args| async move { Ok(Value::Null) },
23216        );
23217
23218        worker.run_until(async {}).await.expect("run worker");
23219    }
23220
23221    #[tokio::test]
23222    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
23223        let server = MockWorkerServer::start();
23224        let client = Client::builder(server.base_url())
23225            .timeout(Duration::from_secs(2))
23226            .build()
23227            .expect("client");
23228        let mut worker = Worker::new(client, "rust-workers")
23229            .worker_id("workflow-only-worker")
23230            .poll_timeout(Duration::from_millis(10));
23231
23232        worker.register_workflow(
23233            "workflow.only",
23234            |_ctx, _input| async move { Ok(Value::Null) },
23235        );
23236
23237        worker.run_until(async {}).await.expect("run worker");
23238    }
23239
23240    #[tokio::test]
23241    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
23242        let server = MockWorkerServer::start();
23243        let client = Client::builder(server.base_url())
23244            .timeout(Duration::from_secs(2))
23245            .build()
23246            .expect("client");
23247        let observations = Arc::new(Mutex::new(Vec::new()));
23248        let observed = Arc::clone(&observations);
23249        let mut worker = Worker::new(client, "rust-workers")
23250            .worker_id("observed-heartbeat-worker")
23251            .poll_timeout(Duration::from_millis(10))
23252            .on_worker_heartbeat(move |observation| {
23253                observed
23254                    .lock()
23255                    .expect("heartbeat observations")
23256                    .push(observation.clone());
23257            });
23258
23259        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
23260            Ok(Value::Null)
23261        });
23262        let acknowledged = Arc::clone(&observations);
23263        worker
23264            .run_until(async move {
23265                tokio::time::timeout(Duration::from_secs(2), async move {
23266                    loop {
23267                        if !acknowledged
23268                            .lock()
23269                            .expect("heartbeat observations")
23270                            .is_empty()
23271                        {
23272                            break;
23273                        }
23274                        tokio::time::sleep(Duration::from_millis(1)).await;
23275                    }
23276                })
23277                .await
23278                .expect("heartbeat acknowledgement within timeout");
23279            })
23280            .await
23281            .expect("run worker");
23282
23283        let observations = observations.lock().expect("heartbeat observations");
23284        let first = observations.first().expect("heartbeat acknowledgement");
23285        assert_eq!(first.worker_id, "observed-heartbeat-worker");
23286        assert_eq!(first.task_queue, "rust-workers");
23287        assert!(first.acknowledged_at_unix_millis > 0);
23288        assert_eq!(first.acknowledgement, json!({}));
23289    }
23290
23291    #[tokio::test]
23292    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
23293        let server = MockWorkerServer::delayed_heartbeat_worker();
23294        let client = Client::builder(server.base_url())
23295            .timeout(Duration::from_secs(3))
23296            .build()
23297            .expect("client");
23298        let observations = Arc::new(Mutex::new(Vec::new()));
23299        let observed = Arc::clone(&observations);
23300        let mut worker = Worker::new(client, "rust-snapshot-workers")
23301            .worker_id("rust-snapshot-worker")
23302            .poll_timeout(Duration::from_millis(10))
23303            .on_worker_heartbeat(move |observation| {
23304                observed
23305                    .lock()
23306                    .expect("heartbeat observations")
23307                    .push(observation.clone());
23308            });
23309
23310        worker.register_workflow("snapshot", |ctx, _input| async move {
23311            ctx.wait_signal("finish").await?;
23312            Ok(json!({"status": "finished"}))
23313        });
23314        worker.register_query("snapshot", "current", |ctx, _args| async move {
23315            Ok(json!(ctx
23316                .signals("increment")
23317                .iter()
23318                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23319                .sum::<i64>()))
23320        });
23321        worker.register_activity("cancel-aware", |_ctx, _args| async move {
23322            Ok(json!({"late": "completion"}))
23323        });
23324
23325        worker
23326            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
23327            .await
23328            .expect("delayed heartbeat must allow a clean worker shutdown");
23329
23330        let observations = observations.lock().expect("heartbeat observations");
23331        assert!(
23332            observations.len() >= 3,
23333            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
23334        );
23335        assert!(
23336            observations.windows(2).all(|pair| {
23337                pair[1].acknowledged_at_unix_millis
23338                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
23339                    >= 850
23340            }),
23341            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
23342        );
23343        drop(observations);
23344
23345        let heartbeat_times = server.request_times("/api/worker/heartbeat");
23346        let delayed_request_at = *heartbeat_times
23347            .get(1)
23348            .expect("intentionally delayed heartbeat request");
23349        let delay_window_start = delayed_request_at + Duration::from_millis(100);
23350        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
23351        for path in [
23352            "/api/worker/workflow-tasks/poll",
23353            "/api/worker/activity-tasks/poll",
23354            "/api/worker/query-tasks/poll",
23355        ] {
23356            assert!(
23357                server
23358                    .request_times(path)
23359                    .iter()
23360                    .any(|received_at| *received_at >= delay_window_start
23361                        && *received_at <= delay_window_end),
23362                "{path} must keep polling while a heartbeat acknowledgement is delayed"
23363            );
23364        }
23365        assert!(
23366            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
23367            "workflow work must be settled"
23368        );
23369        assert!(
23370            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
23371            "activity work must be settled"
23372        );
23373        assert!(
23374            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
23375            "query work must be settled"
23376        );
23377    }
23378
23379    #[tokio::test]
23380    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
23381        let server = MockWorkerServer::heartbeat_retry_worker();
23382        let client = Client::builder(server.base_url())
23383            .timeout(Duration::from_secs(2))
23384            .build()
23385            .expect("client");
23386        let observations = Arc::new(Mutex::new(Vec::new()));
23387        let observed = Arc::clone(&observations);
23388        let worker = Worker::new(client, "rust-workers")
23389            .worker_id("heartbeat-retry-worker")
23390            .retry_policy(WorkerRetryPolicy {
23391                max_retries: 1,
23392                initial_backoff: Duration::from_millis(300),
23393                max_backoff: Duration::from_millis(300),
23394            })
23395            .on_worker_heartbeat(move |observation| {
23396                observed
23397                    .lock()
23398                    .expect("heartbeat observations")
23399                    .push(observation.clone());
23400            });
23401
23402        worker
23403            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
23404            .await
23405            .expect("retryable heartbeat failure must remain bounded and recover");
23406
23407        let observations = observations.lock().expect("heartbeat observations");
23408        assert!(observations.len() >= 3, "heartbeat retry must recover");
23409        assert!(
23410            observations.windows(2).all(|pair| {
23411                pair[1]
23412                    .acknowledged_at_unix_millis
23413                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
23414                    >= 850
23415            }),
23416            "a successful retry must start a fresh advertised cadence: {observations:?}"
23417        );
23418        assert_eq!(
23419            server.request_count("/api/worker/heartbeat"),
23420            observations.len() + 1,
23421            "one retryable failure must add exactly one bounded request"
23422        );
23423    }
23424
23425    #[tokio::test]
23426    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
23427        let server = MockWorkerServer::waiting_query_worker();
23428        let client = Client::builder(server.base_url())
23429            .timeout(Duration::from_secs(2))
23430            .build()
23431            .expect("client");
23432        let observations = Arc::new(Mutex::new(Vec::new()));
23433        let observed = Arc::clone(&observations);
23434        let mut worker = Worker::new(client, "rust-snapshot-workers")
23435            .worker_id("rust-snapshot-worker")
23436            .poll_timeout(Duration::from_millis(10))
23437            .on_worker_heartbeat(move |observation| {
23438                observed
23439                    .lock()
23440                    .expect("heartbeat observations")
23441                    .push(observation.clone());
23442            });
23443
23444        worker.register_workflow("snapshot", |ctx, _input| async move {
23445            ctx.wait_signal("finish").await?;
23446            Ok(json!({"status": "finished"}))
23447        });
23448        worker.register_query("snapshot", "current", |ctx, _args| async move {
23449            let current = ctx
23450                .signals("increment")
23451                .iter()
23452                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23453                .sum::<i64>();
23454            Ok(json!(current))
23455        });
23456        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
23457
23458        worker
23459            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
23460            .await
23461            .expect("pending workflow and query poller must remain live until shutdown");
23462
23463        assert!(
23464            observations.lock().expect("heartbeat observations").len() >= 4,
23465            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
23466        );
23467        assert!(
23468            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
23469            "workflow polling must continue after empty replay acknowledgements"
23470        );
23471        assert!(
23472            server.request_count("/api/worker/query-tasks/poll") >= 2,
23473            "query polling must continue after serving the current query"
23474        );
23475        assert_eq!(
23476            server.request_body("/api/worker/register")["capabilities"],
23477            json!([
23478                CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
23479                DURABLE_SELECTION_CAPABILITY,
23480                MEMO_UPSERTS_CAPABILITY,
23481                TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
23482                QUERY_TASKS_CAPABILITY,
23483                WORKFLOW_UPDATES_CAPABILITY,
23484                MESSAGE_STREAMS_CAPABILITY
23485            ])
23486        );
23487        assert_eq!(
23488            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
23489            json!({
23490                "queries": ["current"],
23491                "query_contracts": [],
23492                "signals": [],
23493                "signal_contracts": [],
23494                "updates": ["replace"],
23495                "update_contracts": [],
23496                "update_validators": [],
23497            })
23498        );
23499
23500        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
23501        assert_eq!(
23502            opened["commands"],
23503            json!([{
23504                "type": "open_signal_wait",
23505                "signal_name": "finish",
23506            }])
23507        );
23508
23509        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
23510            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
23511            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
23512            let failure = server.request_body(&fail_path);
23513            assert_eq!(
23514                failure["failure"]["type"],
23515                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
23516            );
23517            assert_eq!(server.request_count(&completion_path), 0);
23518        }
23519
23520        let query_completion =
23521            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
23522        assert_eq!(query_completion["result"], json!(8));
23523
23524        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
23525        assert_eq!(
23526            server.request_count(terminal_path),
23527            1,
23528            "the matching signal must settle the workflow exactly once"
23529        );
23530        let terminal = server.request_body(terminal_path);
23531        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
23532        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
23533        assert_eq!(
23534            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
23535                .expect("terminal workflow result"),
23536            json!({"status": "finished"})
23537        );
23538    }
23539
23540    #[tokio::test]
23541    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
23542        let server = MockWorkerServer::transient_worker_failures();
23543        let client = Client::builder(server.base_url())
23544            .timeout(Duration::from_secs(2))
23545            .build()
23546            .expect("client");
23547        let mut worker = Worker::new(client, "rust-workers")
23548            .worker_id("retry-worker")
23549            .poll_timeout(Duration::from_millis(10))
23550            .retry_policy(WorkerRetryPolicy {
23551                max_retries: 2,
23552                initial_backoff: Duration::from_millis(1),
23553                max_backoff: Duration::from_millis(1),
23554            });
23555        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23556        worker.register_activity(
23557            "counter.activity",
23558            |_ctx, _input| async move { Ok(Value::Null) },
23559        );
23560        worker.register_query(
23561            "counter",
23562            "current",
23563            |_ctx, _args| async move { Ok(json!(8)) },
23564        );
23565
23566        worker
23567            .run_until(tokio::time::sleep(Duration::from_millis(75)))
23568            .await
23569            .expect("transient failures must not stop the worker");
23570
23571        for path in [
23572            "/api/worker/heartbeat",
23573            "/api/worker/workflow-tasks/poll",
23574            "/api/worker/activity-tasks/poll",
23575            "/api/worker/query-tasks/poll",
23576        ] {
23577            assert!(
23578                server.request_count(path) >= 2,
23579                "{path} must continue after its transient failure"
23580            );
23581        }
23582    }
23583
23584    #[tokio::test]
23585    async fn worker_continues_after_long_poll_capacity_backpressure() {
23586        let server = MockWorkerServer::capacity_limited_activity_poll();
23587        let client = Client::builder(server.base_url())
23588            .timeout(Duration::from_secs(2))
23589            .build()
23590            .expect("client");
23591        let mut worker = Worker::new(client, "rust-workers")
23592            .worker_id("capacity-worker")
23593            .poll_timeout(Duration::from_millis(10))
23594            .retry_policy(WorkerRetryPolicy {
23595                max_retries: 0,
23596                initial_backoff: Duration::from_millis(1),
23597                max_backoff: Duration::from_millis(1),
23598            });
23599        worker.register_activity("capacity.activity", |_ctx, _input| async move {
23600            Ok(json!({"handled": true}))
23601        });
23602
23603        worker
23604            .run_until(tokio::time::sleep(Duration::from_millis(50)))
23605            .await
23606            .expect("capacity backpressure must not stop the worker");
23607
23608        assert!(
23609            server.request_count("/api/worker/activity-tasks/poll") >= 2,
23610            "the activity poller must continue after capacity backpressure"
23611        );
23612        assert_eq!(
23613            server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
23614            1,
23615            "the worker must complete work returned after capacity recovers"
23616        );
23617    }
23618
23619    #[test]
23620    fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
23621        let capacity = Error::Http {
23622            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
23623            body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
23624        };
23625        assert_eq!(
23626            worker_poll_capacity_retry_after(&capacity),
23627            Some(Duration::from_secs(3))
23628        );
23629
23630        let rejected_capacity = Error::Http {
23631            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
23632            body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
23633        };
23634        assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
23635        assert!(!worker_operation_is_retryable(&rejected_capacity));
23636
23637        let ordinary_rate_limit = Error::Http {
23638            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
23639            body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
23640                .to_string(),
23641        };
23642        assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
23643        assert!(worker_operation_is_retryable(&ordinary_rate_limit));
23644    }
23645
23646    #[tokio::test]
23647    async fn worker_bounds_transport_retries() {
23648        let server = MockWorkerServer::unavailable_polls();
23649        let client = Client::builder(server.base_url())
23650            .timeout(Duration::from_secs(2))
23651            .build()
23652            .expect("client");
23653        let mut worker = Worker::new(client, "rust-workers")
23654            .worker_id("bounded-retry-worker")
23655            .poll_timeout(Duration::from_millis(10))
23656            .retry_policy(WorkerRetryPolicy {
23657                max_retries: 2,
23658                initial_backoff: Duration::from_millis(1),
23659                max_backoff: Duration::from_millis(1),
23660            });
23661        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23662
23663        let error = worker.run().await.expect_err("retry bound must terminate");
23664        assert!(matches!(error, Error::Transport(_)));
23665        assert_eq!(
23666            server.request_count("/api/worker/workflow-tasks/poll"),
23667            3,
23668            "one initial request plus exactly two retries"
23669        );
23670    }
23671
23672    #[tokio::test]
23673    async fn worker_retry_policy_can_disable_poll_retries() {
23674        let server = MockWorkerServer::unavailable_polls();
23675        let client = Client::builder(server.base_url())
23676            .timeout(Duration::from_secs(2))
23677            .build()
23678            .expect("client");
23679        let mut worker = Worker::new(client, "rust-workers")
23680            .worker_id("no-retry-worker")
23681            .poll_timeout(Duration::from_millis(10))
23682            .retry_policy(WorkerRetryPolicy {
23683                max_retries: 0,
23684                initial_backoff: Duration::from_millis(1),
23685                max_backoff: Duration::from_millis(1),
23686            });
23687        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23688
23689        let error = worker
23690            .run_once()
23691            .await
23692            .expect_err("disabled retries must return the first transport failure");
23693        assert!(matches!(error, Error::Transport(_)));
23694        assert_eq!(
23695            server.request_count("/api/worker/workflow-tasks/poll"),
23696            1,
23697            "max_retries=0 must send only the initial request"
23698        );
23699    }
23700
23701    #[tokio::test]
23702    async fn worker_does_not_retry_authentication_failures() {
23703        let server = MockWorkerServer::unauthorized_polls();
23704        let client = Client::builder(server.base_url())
23705            .timeout(Duration::from_secs(2))
23706            .build()
23707            .expect("client");
23708        let mut worker = Worker::new(client, "rust-workers")
23709            .worker_id("unauthorized-worker")
23710            .poll_timeout(Duration::from_millis(10));
23711        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23712
23713        let error = worker
23714            .run()
23715            .await
23716            .expect_err("authentication must terminate");
23717        let Error::Http { status, body } = error else {
23718            panic!("expected stable HTTP authentication error");
23719        };
23720        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
23721        assert!(body.contains("authentication_failed"));
23722        assert_eq!(
23723            server.request_count("/api/worker/workflow-tasks/poll"),
23724            1,
23725            "authentication failures must not be retried"
23726        );
23727    }
23728
23729    #[derive(Clone, Debug)]
23730    struct CapturedRequest {
23731        method: String,
23732        path: String,
23733        authorization: Option<String>,
23734        namespace: Option<String>,
23735        worker_protocol: Option<String>,
23736        control_protocol: Option<String>,
23737        body: String,
23738        received_at: Instant,
23739    }
23740
23741    struct MockWorkerServer {
23742        addr: SocketAddr,
23743        stop: Arc<AtomicBool>,
23744        requests: Arc<Mutex<Vec<CapturedRequest>>>,
23745        thread: Option<thread::JoinHandle<()>>,
23746    }
23747
23748    #[derive(Clone, Copy, Default)]
23749    struct MockWorkerBehavior {
23750        reject_query_protocol: bool,
23751        reject_query_completion: bool,
23752        waiting_query_worker: bool,
23753        decline_registration: bool,
23754        complete_named_signal: bool,
23755        poll_failures_per_path: usize,
23756        long_poll_capacity_responses_per_path: usize,
23757        heartbeat_failures: usize,
23758        heartbeat_failure_request: Option<usize>,
23759        delayed_heartbeat_request: Option<usize>,
23760        heartbeat_response_delay: Duration,
23761        concurrent_requests: bool,
23762        unauthorized_polls: bool,
23763        reject_registration: bool,
23764        reject_registration_protocol: bool,
23765        reject_deregistration: bool,
23766        reject_deregistration_protocol: bool,
23767        cancelled_activity: bool,
23768        draining_polls: bool,
23769        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
23770        workflow_completion_status: Option<&'static str>,
23771        workflow_completion_body: Option<&'static str>,
23772    }
23773
23774    impl MockWorkerServer {
23775        fn start() -> Self {
23776            Self::start_with_behavior(MockWorkerBehavior::default())
23777        }
23778
23779        fn reject_query_protocol() -> Self {
23780            Self::start_with_behavior(MockWorkerBehavior {
23781                reject_query_protocol: true,
23782                ..MockWorkerBehavior::default()
23783            })
23784        }
23785
23786        fn reject_query_completion() -> Self {
23787            Self::start_with_behavior(MockWorkerBehavior {
23788                reject_query_completion: true,
23789                ..MockWorkerBehavior::default()
23790            })
23791        }
23792
23793        fn waiting_query_worker() -> Self {
23794            Self::start_with_behavior(MockWorkerBehavior {
23795                waiting_query_worker: true,
23796                complete_named_signal: true,
23797                ..MockWorkerBehavior::default()
23798            })
23799        }
23800
23801        fn transient_worker_failures() -> Self {
23802            Self::start_with_behavior(MockWorkerBehavior {
23803                poll_failures_per_path: 1,
23804                heartbeat_failures: 1,
23805                ..MockWorkerBehavior::default()
23806            })
23807        }
23808
23809        fn consecutive_poll_failures(count: usize) -> Self {
23810            Self::start_with_behavior(MockWorkerBehavior {
23811                poll_failures_per_path: count,
23812                ..MockWorkerBehavior::default()
23813            })
23814        }
23815
23816        fn capacity_limited_activity_poll() -> Self {
23817            Self::start_with_behavior(MockWorkerBehavior {
23818                long_poll_capacity_responses_per_path: 1,
23819                ..MockWorkerBehavior::default()
23820            })
23821        }
23822
23823        fn delayed_heartbeat_worker() -> Self {
23824            Self::start_with_behavior(MockWorkerBehavior {
23825                waiting_query_worker: true,
23826                delayed_heartbeat_request: Some(2),
23827                heartbeat_response_delay: Duration::from_millis(1_500),
23828                concurrent_requests: true,
23829                cancelled_activity: true,
23830                ..MockWorkerBehavior::default()
23831            })
23832        }
23833
23834        fn heartbeat_retry_worker() -> Self {
23835            Self::start_with_behavior(MockWorkerBehavior {
23836                waiting_query_worker: true,
23837                heartbeat_failure_request: Some(2),
23838                concurrent_requests: true,
23839                ..MockWorkerBehavior::default()
23840            })
23841        }
23842
23843        fn unavailable_polls() -> Self {
23844            Self::start_with_behavior(MockWorkerBehavior {
23845                poll_failures_per_path: usize::MAX,
23846                ..MockWorkerBehavior::default()
23847            })
23848        }
23849
23850        fn unauthorized_polls() -> Self {
23851            Self::start_with_behavior(MockWorkerBehavior {
23852                unauthorized_polls: true,
23853                ..MockWorkerBehavior::default()
23854            })
23855        }
23856
23857        fn rejected_registration() -> Self {
23858            Self::start_with_behavior(MockWorkerBehavior {
23859                reject_registration: true,
23860                ..MockWorkerBehavior::default()
23861            })
23862        }
23863
23864        fn rejected_registration_protocol() -> Self {
23865            Self::start_with_behavior(MockWorkerBehavior {
23866                reject_registration_protocol: true,
23867                ..MockWorkerBehavior::default()
23868            })
23869        }
23870
23871        fn declined_registration() -> Self {
23872            Self::start_with_behavior(MockWorkerBehavior {
23873                decline_registration: true,
23874                ..MockWorkerBehavior::default()
23875            })
23876        }
23877
23878        fn rejected_deregistration() -> Self {
23879            Self::start_with_behavior(MockWorkerBehavior {
23880                reject_deregistration: true,
23881                ..MockWorkerBehavior::default()
23882            })
23883        }
23884
23885        fn rejected_deregistration_protocol() -> Self {
23886            Self::start_with_behavior(MockWorkerBehavior {
23887                reject_deregistration_protocol: true,
23888                ..MockWorkerBehavior::default()
23889            })
23890        }
23891
23892        fn unauthorized_polls_and_rejected_deregistration() -> Self {
23893            Self::start_with_behavior(MockWorkerBehavior {
23894                unauthorized_polls: true,
23895                reject_deregistration: true,
23896                ..MockWorkerBehavior::default()
23897            })
23898        }
23899
23900        fn cancelled_activity() -> Self {
23901            Self::start_with_behavior(MockWorkerBehavior {
23902                cancelled_activity: true,
23903                ..MockWorkerBehavior::default()
23904            })
23905        }
23906
23907        fn draining_polls() -> Self {
23908            Self::start_with_behavior(MockWorkerBehavior {
23909                draining_polls: true,
23910                ..MockWorkerBehavior::default()
23911            })
23912        }
23913
23914        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
23915            Self::start_with_behavior(MockWorkerBehavior {
23916                invalid_task_payload_codec: Some(codec),
23917                ..MockWorkerBehavior::default()
23918            })
23919        }
23920
23921        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
23922            Self::start_with_behavior(MockWorkerBehavior {
23923                workflow_completion_status: Some(status),
23924                workflow_completion_body: Some(body),
23925                ..MockWorkerBehavior::default()
23926            })
23927        }
23928
23929        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
23930            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
23931            listener
23932                .set_nonblocking(true)
23933                .expect("configure mock listener");
23934            let addr = listener.local_addr().expect("mock server address");
23935            let stop = Arc::new(AtomicBool::new(false));
23936            let server_stop = Arc::clone(&stop);
23937            let requests = Arc::new(Mutex::new(Vec::new()));
23938            let server_requests = Arc::clone(&requests);
23939            let thread = thread::spawn(move || {
23940                let mut request_threads = Vec::new();
23941                while !server_stop.load(Ordering::SeqCst) {
23942                    match listener.accept() {
23943                        Ok((mut stream, _)) => {
23944                            if behavior.concurrent_requests {
23945                                let requests = Arc::clone(&server_requests);
23946                                request_threads.push(thread::spawn(move || {
23947                                    handle_mock_worker_request(&mut stream, &requests, behavior)
23948                                }));
23949                            } else {
23950                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
23951                            }
23952                        }
23953                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
23954                            let mut index = 0;
23955                            while index < request_threads.len() {
23956                                if request_threads[index].is_finished() {
23957                                    request_threads
23958                                        .swap_remove(index)
23959                                        .join()
23960                                        .expect("join mock request");
23961                                } else {
23962                                    index += 1;
23963                                }
23964                            }
23965                            thread::sleep(Duration::from_millis(5));
23966                        }
23967                        Err(_) => break,
23968                    }
23969                }
23970                for request_thread in request_threads {
23971                    request_thread.join().expect("join mock request");
23972                }
23973            });
23974
23975            Self {
23976                addr,
23977                stop,
23978                requests,
23979                thread: Some(thread),
23980            }
23981        }
23982
23983        fn base_url(&self) -> String {
23984            format!("http://{}", self.addr)
23985        }
23986
23987        fn worker_protocol_for(&self, path: &str) -> Option<String> {
23988            self.requests
23989                .lock()
23990                .expect("captured requests")
23991                .iter()
23992                .find(|request| request.path == path)
23993                .and_then(|request| request.worker_protocol.clone())
23994        }
23995
23996        fn control_protocol_for(&self, path: &str) -> Option<String> {
23997            self.requests
23998                .lock()
23999                .expect("captured requests")
24000                .iter()
24001                .find(|request| request.path == path)
24002                .and_then(|request| request.control_protocol.clone())
24003        }
24004
24005        fn method_for(&self, path: &str) -> Option<String> {
24006            self.requests
24007                .lock()
24008                .expect("captured requests")
24009                .iter()
24010                .find(|request| request.path == path)
24011                .map(|request| request.method.clone())
24012        }
24013
24014        fn authorization_for(&self, path: &str) -> Option<String> {
24015            self.requests
24016                .lock()
24017                .expect("captured requests")
24018                .iter()
24019                .find(|request| request.path == path)
24020                .and_then(|request| request.authorization.clone())
24021        }
24022
24023        fn namespace_for(&self, path: &str) -> Option<String> {
24024            self.requests
24025                .lock()
24026                .expect("captured requests")
24027                .iter()
24028                .find(|request| request.path == path)
24029                .and_then(|request| request.namespace.clone())
24030        }
24031
24032        fn request_count(&self, path: &str) -> usize {
24033            self.requests
24034                .lock()
24035                .expect("captured requests")
24036                .iter()
24037                .filter(|request| request.path == path)
24038                .count()
24039        }
24040
24041        fn captured_paths(&self) -> Vec<String> {
24042            self.requests
24043                .lock()
24044                .expect("captured requests")
24045                .iter()
24046                .map(|request| request.path.clone())
24047                .collect()
24048        }
24049
24050        fn request_times(&self, path: &str) -> Vec<Instant> {
24051            self.requests
24052                .lock()
24053                .expect("captured requests")
24054                .iter()
24055                .filter(|request| request.path == path)
24056                .map(|request| request.received_at)
24057                .collect()
24058        }
24059
24060        fn request_body(&self, path: &str) -> Value {
24061            let requests = self.requests.lock().expect("captured requests");
24062            let body = &requests
24063                .iter()
24064                .find(|request| request.path == path)
24065                .unwrap_or_else(|| panic!("missing request for {path}"))
24066                .body;
24067            serde_json::from_str(body).unwrap_or_else(|error| {
24068                panic!("invalid JSON request body for {path}: {error}: {body:?}")
24069            })
24070        }
24071
24072        fn request_bodies(&self, path: &str) -> Vec<Value> {
24073            self.requests
24074                .lock()
24075                .expect("captured requests")
24076                .iter()
24077                .filter(|request| request.path == path)
24078                .map(|request| {
24079                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
24080                        panic!(
24081                            "invalid JSON request body for {path}: {error}: {:?}",
24082                            request.body
24083                        )
24084                    })
24085                })
24086                .collect()
24087        }
24088    }
24089
24090    impl Drop for MockWorkerServer {
24091        fn drop(&mut self) {
24092            self.stop.store(true, Ordering::SeqCst);
24093            let _ = TcpStream::connect(self.addr);
24094
24095            if let Some(thread) = self.thread.take() {
24096                thread.join().expect("join mock server");
24097            }
24098        }
24099    }
24100
24101    fn handle_mock_worker_request(
24102        stream: &mut TcpStream,
24103        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
24104        behavior: MockWorkerBehavior,
24105    ) {
24106        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
24107        let mut buffer = [0_u8; 8192];
24108        let mut request = Vec::new();
24109
24110        loop {
24111            match stream.read(&mut buffer) {
24112                Ok(0) => break,
24113                Ok(read) => {
24114                    request.extend_from_slice(&buffer[..read]);
24115                    if mock_request_is_complete(&request) {
24116                        break;
24117                    }
24118                }
24119                Err(error)
24120                    if matches!(
24121                        error.kind(),
24122                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
24123                    ) =>
24124                {
24125                    break;
24126                }
24127                Err(_) => return,
24128            }
24129        }
24130
24131        let request = String::from_utf8_lossy(&request);
24132        let body = request
24133            .split_once("\r\n\r\n")
24134            .map(|(_, body)| body)
24135            .unwrap_or_default();
24136        let path = request
24137            .lines()
24138            .next()
24139            .and_then(|line| line.split_whitespace().nth(1))
24140            .unwrap_or_default();
24141        let method = request
24142            .lines()
24143            .next()
24144            .and_then(|line| line.split_whitespace().next())
24145            .unwrap_or_default();
24146        let authorization = request.lines().find_map(|line| {
24147            let (name, value) = line.split_once(':')?;
24148            name.eq_ignore_ascii_case("Authorization")
24149                .then(|| value.trim().to_string())
24150        });
24151        let namespace = request.lines().find_map(|line| {
24152            let (name, value) = line.split_once(':')?;
24153            name.eq_ignore_ascii_case("X-Namespace")
24154                .then(|| value.trim().to_string())
24155        });
24156        let worker_protocol = request.lines().find_map(|line| {
24157            let (name, value) = line.split_once(':')?;
24158            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
24159                .then(|| value.trim().to_string())
24160        });
24161        let control_protocol = request.lines().find_map(|line| {
24162            let (name, value) = line.split_once(':')?;
24163            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
24164                .then(|| value.trim().to_string())
24165        });
24166        let request_number = {
24167            let mut requests = requests.lock().expect("captured requests");
24168            requests.push(CapturedRequest {
24169                method: method.to_string(),
24170                path: path.to_string(),
24171                authorization,
24172                namespace,
24173                worker_protocol: worker_protocol.clone(),
24174                control_protocol,
24175                body: body.to_string(),
24176                received_at: Instant::now(),
24177            });
24178            requests
24179                .iter()
24180                .filter(|request| request.path == path)
24181                .count()
24182        };
24183
24184        if path == "/api/worker/register" {
24185            if behavior.reject_registration_protocol {
24186                write_mock_response(
24187                    stream,
24188                    "400 Bad Request",
24189                    r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
24190                );
24191                return;
24192            }
24193            if behavior.reject_registration {
24194                write_mock_response(
24195                    stream,
24196                    "503 Service Unavailable",
24197                    r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
24198                );
24199                return;
24200            }
24201        }
24202
24203        if path.starts_with("/api/worker/registrations/") {
24204            if behavior.reject_deregistration_protocol {
24205                write_mock_response(
24206                    stream,
24207                    "400 Bad Request",
24208                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
24209                );
24210            } else if behavior.reject_deregistration {
24211                write_mock_response(
24212                    stream,
24213                    "403 Forbidden",
24214                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
24215                );
24216            } else {
24217                write_mock_response(
24218                    stream,
24219                    "200 OK",
24220                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
24221                );
24222            }
24223            return;
24224        }
24225
24226        let is_poll = matches!(
24227            path,
24228            "/api/worker/workflow-tasks/poll"
24229                | "/api/worker/activity-tasks/poll"
24230                | "/api/worker/query-tasks/poll"
24231        );
24232        if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
24233            write_mock_response(
24234                stream,
24235                "429 Too Many Requests",
24236                r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
24237            );
24238            return;
24239        }
24240        if is_poll && request_number <= behavior.poll_failures_per_path {
24241            return;
24242        }
24243        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
24244            return;
24245        }
24246        if path == "/api/worker/heartbeat"
24247            && behavior.heartbeat_failure_request == Some(request_number)
24248        {
24249            return;
24250        }
24251        if path == "/api/worker/heartbeat"
24252            && behavior.delayed_heartbeat_request == Some(request_number)
24253        {
24254            thread::sleep(behavior.heartbeat_response_delay);
24255        }
24256        if behavior.unauthorized_polls && is_poll {
24257            write_mock_response(
24258                stream,
24259                "401 Unauthorized",
24260                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
24261            );
24262            return;
24263        }
24264        if behavior.draining_polls && is_poll {
24265            write_mock_response(
24266                stream,
24267                "409 Conflict",
24268                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
24269            );
24270            return;
24271        }
24272
24273        if let Some(codec_case) = behavior.invalid_task_payload_codec {
24274            if is_poll && request_number == 1 {
24275                let mut task = match path {
24276                    "/api/worker/workflow-tasks/poll" => json!({
24277                        "task_id": "codec-workflow",
24278                        "workflow_type": "codec.workflow",
24279                        "payload_codec": DEFAULT_CODEC,
24280                        "workflow_task_attempt": 1,
24281                        "lease_owner": "codec-worker"
24282                    }),
24283                    "/api/worker/activity-tasks/poll" => json!({
24284                        "task_id": "codec-activity",
24285                        "activity_attempt_id": "codec-activity-attempt",
24286                        "activity_type": "codec.activity",
24287                        "payload_codec": DEFAULT_CODEC,
24288                        "attempt_number": 1,
24289                        "lease_owner": "codec-worker"
24290                    }),
24291                    "/api/worker/query-tasks/poll" => json!({
24292                        "query_task_id": "codec-query",
24293                        "query_task_attempt": 1,
24294                        "workflow_type": "codec.workflow",
24295                        "query_name": "known",
24296                        "payload_codec": DEFAULT_CODEC,
24297                        "lease_owner": "codec-worker"
24298                    }),
24299                    _ => unreachable!("is_poll limits task codec probe paths"),
24300                };
24301                codec_case.apply(&mut task);
24302                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
24303                return;
24304            }
24305
24306            if matches!(
24307                path,
24308                "/api/worker/workflow-tasks/codec-workflow/fail"
24309                    | "/api/worker/activity-tasks/codec-activity/fail"
24310                    | "/api/worker/query-tasks/codec-query/fail"
24311            ) {
24312                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
24313                return;
24314            }
24315        }
24316
24317        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
24318            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
24319            let body = format!(
24320                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
24321            );
24322            write_mock_response(stream, "400 Bad Request", &body);
24323            return;
24324        }
24325
24326        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
24327        {
24328            write_mock_response(
24329                stream,
24330                "409 Conflict",
24331                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
24332            );
24333            return;
24334        }
24335
24336        if behavior.workflow_completion_status.is_some()
24337            && path == "/api/worker/workflow-tasks/poll"
24338            && request_number == 1
24339        {
24340            write_mock_response(
24341                stream,
24342                "200 OK",
24343                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
24344            );
24345            return;
24346        }
24347
24348        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
24349            if let (Some(status), Some(body)) = (
24350                behavior.workflow_completion_status,
24351                behavior.workflow_completion_body,
24352            ) {
24353                write_mock_response(stream, status, body);
24354                return;
24355            }
24356        }
24357
24358        if behavior.waiting_query_worker {
24359            if behavior.complete_named_signal
24360                && path == "/api/worker/workflow-tasks/poll"
24361                && request_number == 1
24362            {
24363                let body = json!({
24364                    "task": {
24365                        "task_id": "snapshot-open",
24366                        "workflow_id": "snapshot-1",
24367                        "run_id": "snapshot-run-1",
24368                        "workflow_type": "snapshot",
24369                        "payload_codec": DEFAULT_CODEC,
24370                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24371                            .expect("Avro workflow arguments"),
24372                        "history_events": [],
24373                        "workflow_task_attempt": 1,
24374                        "lease_owner": "rust-snapshot-worker"
24375                    }
24376                })
24377                .to_string();
24378                write_mock_response(stream, "200 OK", &body);
24379                return;
24380            }
24381
24382            let signal_request = request_number - usize::from(behavior.complete_named_signal);
24383            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
24384            if path == "/api/worker/workflow-tasks/poll"
24385                && signal_request >= 1
24386                && signal_request <= signal_request_limit
24387            {
24388                let finish = behavior.complete_named_signal && signal_request == 3;
24389                let amounts = if signal_request == 1 {
24390                    vec![3]
24391                } else {
24392                    vec![3, 5]
24393                };
24394                let task_id = if signal_request == 1 {
24395                    "snapshot-wait-3"
24396                } else if finish {
24397                    "snapshot-finish"
24398                } else {
24399                    "snapshot-wait-5"
24400                };
24401                let mut history_events = std::iter::once(json!({
24402                    "event_type": "SignalWaitOpened",
24403                    "payload": {"sequence": 1, "signal_name": "finish"}
24404                }))
24405                .chain(amounts.iter().enumerate().map(|(index, amount)| {
24406                    json!({
24407                        "event_type": "SignalReceived",
24408                        "payload": {
24409                            "signal_id": format!("increment-{amount}"),
24410                            "signal_name": "increment",
24411                            "workflow_sequence": index + 2,
24412                            "payload_codec": DEFAULT_CODEC,
24413                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24414                                .expect("Avro signal envelope")
24415                        }
24416                    })
24417                }))
24418                .collect::<Vec<_>>();
24419                let (resume_id, resume_name, resume_arguments) = if finish {
24420                    history_events.push(json!({
24421                        "event_type": "SignalReceived",
24422                        "payload": {
24423                            "signal_id": "finish",
24424                            "signal_name": "finish",
24425                            "workflow_sequence": 4,
24426                            "payload_codec": DEFAULT_CODEC,
24427                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24428                                .expect("Avro finish signal envelope")
24429                        }
24430                    }));
24431                    (
24432                        "finish".to_string(),
24433                        "finish".to_string(),
24434                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
24435                            .expect("Avro finish resume signal"),
24436                    )
24437                } else {
24438                    let amount = amounts.last().expect("amount");
24439                    (
24440                        format!("increment-{amount}"),
24441                        "increment".to_string(),
24442                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24443                            .expect("Avro increment resume signal"),
24444                    )
24445                };
24446                let body = json!({
24447                    "task": {
24448                        "task_id": task_id,
24449                        "workflow_id": "snapshot-1",
24450                        "run_id": "snapshot-run-1",
24451                        "workflow_type": "snapshot",
24452                        "payload_codec": DEFAULT_CODEC,
24453                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24454                            .expect("Avro workflow arguments"),
24455                        "history_events": history_events,
24456                        "workflow_task_attempt": 1,
24457                        "workflow_signal_id": resume_id,
24458                        "signal_name": resume_name,
24459                        "signal_arguments": resume_arguments,
24460                        "lease_owner": "rust-snapshot-worker"
24461                    }
24462                })
24463                .to_string();
24464                write_mock_response(stream, "200 OK", &body);
24465                return;
24466            }
24467
24468            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
24469                let history_events = [3, 5]
24470                    .into_iter()
24471                    .enumerate()
24472                    .map(|(index, amount)| {
24473                        json!({
24474                            "event_type": "SignalReceived",
24475                            "payload": {
24476                                "signal_id": format!("increment-{amount}"),
24477                                "signal_name": "increment",
24478                                "workflow_sequence": index + 2,
24479                                "payload_codec": DEFAULT_CODEC,
24480                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24481                                    .expect("Avro query signal envelope")
24482                            }
24483                        })
24484                    })
24485                    .collect::<Vec<_>>();
24486                let body = json!({
24487                    "task": {
24488                        "query_task_id": "snapshot-current",
24489                        "query_task_attempt": 1,
24490                        "lease_owner": "rust-snapshot-worker",
24491                        "workflow_id": "snapshot-1",
24492                        "run_id": "snapshot-run-1",
24493                        "workflow_type": "snapshot",
24494                        "query_name": "current",
24495                        "payload_codec": DEFAULT_CODEC,
24496                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24497                            .expect("Avro workflow arguments"),
24498                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24499                            .expect("Avro query arguments"),
24500                        "history_events": history_events,
24501                        "run_status": "waiting"
24502                    }
24503                })
24504                .to_string();
24505                write_mock_response(stream, "200 OK", &body);
24506                return;
24507            }
24508
24509            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
24510                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
24511            {
24512                write_mock_response(
24513                    stream,
24514                    "200 OK",
24515                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
24516                );
24517                return;
24518            }
24519
24520            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
24521                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
24522                return;
24523            }
24524
24525            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
24526                write_mock_response(
24527                    stream,
24528                    "200 OK",
24529                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
24530                );
24531                return;
24532            }
24533
24534            if path == "/api/worker/query-tasks/snapshot-current/complete" {
24535                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
24536                return;
24537            }
24538        }
24539
24540        if matches!(
24541            path,
24542            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
24543        ) {
24544            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
24545                .expect("typed mock result");
24546            let body = json!({
24547                "result": typed_fidelity_probe().into_json().expect("result projection"),
24548                "result_envelope": result,
24549            })
24550            .to_string();
24551            write_mock_response(stream, "200 OK", &body);
24552            return;
24553        }
24554
24555        if path == "/api/workflows/typed-1" {
24556            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
24557                .expect("typed mock result");
24558            let body = json!({
24559                "workflow_id": "typed-1",
24560                "run_id": "run-typed-1",
24561                "workflow_type": "typed.echo",
24562                "status": "completed",
24563                "output": typed_fidelity_probe().into_json().expect("output projection"),
24564                "output_envelope": result,
24565            })
24566            .to_string();
24567            write_mock_response(stream, "200 OK", &body);
24568            return;
24569        }
24570
24571        let (status, body) = match path {
24572            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
24573            "/api/workflows" => (
24574                "201 Created",
24575                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
24576            ),
24577            "/api/worker/register" if behavior.decline_registration => (
24578                "200 OK",
24579                r#"{"worker_id":"declined-worker","registered":false}"#,
24580            ),
24581            "/api/worker/register" if behavior.waiting_query_worker => (
24582                "200 OK",
24583                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
24584            ),
24585            "/api/worker/register" => (
24586                "200 OK",
24587                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
24588            ),
24589            "/api/worker/heartbeat" => ("200 OK", "{}"),
24590            "/api/worker/activity-tasks/poll"
24591                if behavior.cancelled_activity && request_number == 1 =>
24592            {
24593                (
24594                    "200 OK",
24595                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
24596                )
24597            }
24598            "/api/worker/activity-tasks/poll"
24599                if behavior.long_poll_capacity_responses_per_path > 0
24600                    && request_number
24601                        == behavior
24602                            .long_poll_capacity_responses_per_path
24603                            .saturating_add(1) =>
24604            {
24605                (
24606                    "200 OK",
24607                    r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
24608                )
24609            }
24610            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
24611                ("200 OK", r#"{"task":null}"#)
24612            }
24613            "/api/worker/query-tasks/poll"
24614                if behavior.reject_query_completion && request_number == 1 =>
24615            {
24616                (
24617                    "200 OK",
24618                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
24619                )
24620            }
24621            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
24622            "/api/worker/query-tasks/query-capture/complete"
24623            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
24624            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
24625                "200 OK",
24626                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
24627            ),
24628            "/api/worker/activity-tasks/activity-cancel/complete" => (
24629                "409 Conflict",
24630                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
24631            ),
24632            "/api/worker/activity-tasks/activity-typed/complete"
24633            | "/api/worker/activity-tasks/activity-typed/fail"
24634            | "/api/worker/activity-tasks/capacity-activity/complete"
24635            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
24636            "/api/workflows/counter-1/query/current" => (
24637                "200 OK",
24638                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
24639            ),
24640            "/api/workflows/counter-1/query/missing" => (
24641                "404 Not Found",
24642                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
24643            ),
24644            "/api/workflows/wf-lifecycle/cancel" => (
24645                "200 OK",
24646                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
24647            ),
24648            "/api/workflows/wf-lifecycle/terminate" => (
24649                "200 OK",
24650                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
24651            ),
24652            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
24653                "200 OK",
24654                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
24655            ),
24656            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
24657                "200 OK",
24658                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
24659            ),
24660            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
24661            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
24662                "409 Conflict",
24663                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
24664            ),
24665            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
24666                "200 OK",
24667                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
24668            ),
24669            "/api/workflows/wf-cancelled" => (
24670                "200 OK",
24671                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
24672            ),
24673            "/api/workflows/wf-terminated" => (
24674                "200 OK",
24675                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
24676            ),
24677            "/api/workflows/wf-timed-out" => (
24678                "200 OK",
24679                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
24680            ),
24681            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
24682                "200 OK",
24683                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
24684            ),
24685            "/api/workflows/wf-selected" => (
24686                "200 OK",
24687                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
24688            ),
24689            "/api/workflows/wf-selected/runs/run-selected" => (
24690                "200 OK",
24691                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
24692            ),
24693            _ => ("404 Not Found", r#"{"message":"not found"}"#),
24694        };
24695        write_mock_response(stream, status, body);
24696    }
24697
24698    fn mock_request_is_complete(request: &[u8]) -> bool {
24699        let Some(header_end) = request
24700            .windows(4)
24701            .position(|window| window == b"\r\n\r\n")
24702            .map(|position| position + 4)
24703        else {
24704            return false;
24705        };
24706        let headers = String::from_utf8_lossy(&request[..header_end]);
24707        let content_length = headers.lines().find_map(|line| {
24708            let (name, value) = line.split_once(':')?;
24709            name.eq_ignore_ascii_case("content-length")
24710                .then(|| value.trim().parse::<usize>().ok())
24711                .flatten()
24712        });
24713
24714        request.len() >= header_end + content_length.unwrap_or(0)
24715    }
24716
24717    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
24718        let response = format!(
24719            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
24720            body.len()
24721        );
24722
24723        let _ = stream.write_all(response.as_bytes());
24724        let _ = stream.flush();
24725    }
24726}